View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.index.examples.indexing;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Collection;
25  import java.util.List;
26  import java.util.Set;
27  
28  import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
29  import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
30  import org.apache.lucene.queryparser.classic.ParseException;
31  import org.apache.lucene.search.BooleanQuery;
32  import org.apache.lucene.search.Query;
33  import org.apache.maven.index.ArtifactContext;
34  import org.apache.maven.index.ArtifactInfo;
35  import org.apache.maven.index.ArtifactScanningListener;
36  import org.apache.maven.index.FlatSearchRequest;
37  import org.apache.maven.index.FlatSearchResponse;
38  import org.apache.maven.index.Indexer;
39  import org.apache.maven.index.MAVEN;
40  import org.apache.maven.index.Scanner;
41  import org.apache.maven.index.ScanningRequest;
42  import org.apache.maven.index.ScanningResult;
43  import org.apache.maven.index.context.IndexCreator;
44  import org.apache.maven.index.context.IndexingContext;
45  import org.apache.maven.index.expr.SourcedSearchExpression;
46  import org.slf4j.Logger;
47  import org.slf4j.LoggerFactory;
48  
49  import static java.util.Arrays.asList;
50  import static org.apache.lucene.search.BooleanClause.Occur.MUST;
51  
52  /**
53   * This class provides means to index and search for artifacts in a repository on the file system.
54   *
55   * @author mtodorov
56   */
57  public class RepositoryIndexer {
58  
59      private static final Logger LOGGER = LoggerFactory.getLogger(RepositoryIndexer.class);
60  
61      private static final String[] LUCENE_FIELDS = new String[] {"g", "a", "v", "p", "c"};
62  
63      private static final WhitespaceAnalyzer LUCENE_ANALYZER = new WhitespaceAnalyzer();
64  
65      private Indexer indexer;
66  
67      private Scanner scanner;
68  
69      private List<IndexCreator> indexers;
70  
71      private IndexingContext indexingContext;
72  
73      private String repositoryId;
74  
75      private File repositoryBasedir;
76  
77      private File indexDir;
78  
79      public RepositoryIndexer() {
80          // no op
81      }
82  
83      public void close() throws IOException {
84          indexer.closeIndexingContext(indexingContext, false);
85      }
86  
87      public void close(boolean deleteFiles) throws IOException {
88          indexingContext.close(deleteFiles);
89      }
90  
91      public void delete(final Collection<ArtifactInfo> artifacts) throws IOException {
92          final List<ArtifactContext> delete = new ArrayList<>();
93          for (final ArtifactInfo artifact : artifacts) {
94              LOGGER.debug(
95                      "Deleting artifact: {}; ctx id: {}; idx dir: {}",
96                      artifact,
97                      indexingContext.getId(),
98                      indexingContext.getIndexDirectory());
99  
100             delete.add(new ArtifactContext(null, null, null, artifact, null));
101         }
102 
103         getIndexer().deleteArtifactsFromIndex(delete, indexingContext);
104     }
105 
106     public Set<ArtifactInfo> search(
107             final String groupId,
108             final String artifactId,
109             final String version,
110             final String packaging,
111             final String classifier)
112             throws IOException {
113         final BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
114 
115         if (groupId != null) {
116             queryBuilder.add(getIndexer().constructQuery(MAVEN.GROUP_ID, new SourcedSearchExpression(groupId)), MUST);
117         }
118 
119         if (artifactId != null) {
120             queryBuilder.add(
121                     getIndexer().constructQuery(MAVEN.ARTIFACT_ID, new SourcedSearchExpression(artifactId)), MUST);
122         }
123 
124         if (version != null) {
125             queryBuilder.add(getIndexer().constructQuery(MAVEN.VERSION, new SourcedSearchExpression(version)), MUST);
126         }
127 
128         if (packaging != null) {
129             queryBuilder.add(
130                     getIndexer().constructQuery(MAVEN.PACKAGING, new SourcedSearchExpression(packaging)), MUST);
131         } else {
132             // Fallback to jar
133             queryBuilder.add(getIndexer().constructQuery(MAVEN.PACKAGING, new SourcedSearchExpression("jar")), MUST);
134         }
135 
136         if (classifier != null) {
137             queryBuilder.add(
138                     getIndexer().constructQuery(MAVEN.CLASSIFIER, new SourcedSearchExpression(classifier)), MUST);
139         }
140 
141         Query query = queryBuilder.build();
142 
143         LOGGER.debug(
144                 "Executing search query: {}; ctx id: {}; idx dir: {}",
145                 query,
146                 indexingContext.getId(),
147                 indexingContext.getIndexDirectory());
148 
149         final FlatSearchResponse response = getIndexer().searchFlat(new FlatSearchRequest(query, indexingContext));
150 
151         LOGGER.info("Hit count: {}", response.getReturnedHitsCount());
152 
153         final Set<ArtifactInfo> results = response.getResults();
154         if (LOGGER.isDebugEnabled()) {
155             for (final ArtifactInfo result : results) {
156                 LOGGER.debug("Found artifact: {}", result.toString());
157             }
158         }
159 
160         return results;
161     }
162 
163     public Set<ArtifactInfo> search(final String queryText) throws ParseException, IOException {
164         final Query query = new MultiFieldQueryParser(LUCENE_FIELDS, LUCENE_ANALYZER).parse(queryText);
165 
166         LOGGER.debug(
167                 "Executing search query: {}; ctx id: {}; idx dir: {}",
168                 query,
169                 indexingContext.getId(),
170                 indexingContext.getIndexDirectory());
171 
172         final FlatSearchResponse response = getIndexer().searchFlat(new FlatSearchRequest(query, indexingContext));
173 
174         final Set<ArtifactInfo> results = response.getResults();
175         if (LOGGER.isDebugEnabled()) {
176             LOGGER.debug("Hit count: {}", response.getReturnedHitsCount());
177 
178             for (final ArtifactInfo result : results) {
179                 LOGGER.debug("Found artifact: {}; uinfo: {}", result.toString(), result.getUinfo());
180             }
181         }
182 
183         return results;
184     }
185 
186     public Set<ArtifactInfo> searchBySHA1(final String checksum) throws IOException {
187         final BooleanQuery query = new BooleanQuery.Builder()
188                 .add(getIndexer().constructQuery(MAVEN.SHA1, new SourcedSearchExpression(checksum)), MUST)
189                 .build();
190 
191         LOGGER.debug(
192                 "Executing search query: {}; ctx id: {}; idx dir: {}",
193                 query,
194                 indexingContext.getId(),
195                 indexingContext.getIndexDirectory());
196 
197         final FlatSearchResponse response = getIndexer().searchFlat(new FlatSearchRequest(query, indexingContext));
198 
199         LOGGER.info("Hit count: {}", response.getReturnedHitsCount());
200 
201         final Set<ArtifactInfo> results = response.getResults();
202         if (LOGGER.isDebugEnabled()) {
203             for (final ArtifactInfo result : results) {
204                 LOGGER.debug("Found artifact: {}", result.toString());
205             }
206         }
207 
208         return results;
209     }
210 
211     public int index(final File startingPath) {
212         final ScanningResult scan = getScanner()
213                 .scan(new ScanningRequest(
214                         indexingContext,
215                         new ReindexArtifactScanningListener(),
216                         startingPath == null ? "." : startingPath.getPath()));
217         return scan.getTotalFiles();
218     }
219 
220     public void addArtifactToIndex(final File artifactFile, final ArtifactInfo artifactInfo) throws IOException {
221         getIndexer()
222                 .addArtifactsToIndex(
223                         asList(new ArtifactContext(null, artifactFile, null, artifactInfo, null)), indexingContext);
224     }
225 
226     public void addArtifactToIndex(
227             String repository,
228             File artifactFile,
229             String groupId,
230             String artifactId,
231             String version,
232             String extension,
233             String classifier)
234             throws IOException {
235         ArtifactInfo artifactInfo = new ArtifactInfo(repository, groupId, artifactId, version, classifier, extension);
236         if (extension != null) {
237             artifactInfo.setFieldValue(MAVEN.EXTENSION, extension);
238         }
239 
240         LOGGER.debug("Adding artifact: {}; repo: {}; type: {}", artifactInfo.getUinfo(), repository, extension);
241 
242         getIndexer()
243                 .addArtifactsToIndex(
244                         asList(new ArtifactContext(
245                                 null, artifactFile, null, artifactInfo, artifactInfo.calculateGav())),
246                         indexingContext);
247     }
248 
249     private class ReindexArtifactScanningListener implements ArtifactScanningListener {
250 
251         int totalFiles = 0;
252 
253         private IndexingContext context;
254 
255         @Override
256         public void scanningStarted(final IndexingContext context) {
257             this.context = context;
258         }
259 
260         @Override
261         public void scanningFinished(final IndexingContext context, final ScanningResult result) {
262             result.setTotalFiles(totalFiles);
263             LOGGER.debug(
264                     "Scanning finished; total files: {}; has exception: {}",
265                     result.getTotalFiles(),
266                     result.hasExceptions());
267         }
268 
269         @Override
270         public void artifactError(final ArtifactContext ac, final Exception ex) {
271             LOGGER.error("Artifact error!", ex);
272         }
273 
274         @Override
275         public void artifactDiscovered(final ArtifactContext ac) {
276             try {
277                 LOGGER.debug(
278                         "Adding artifact gav: {}; ctx id: {}; idx dir: {}",
279                         ac.getGav(),
280                         context.getId(),
281                         context.getIndexDirectory());
282 
283                 getIndexer().addArtifactsToIndex(asList(ac), context);
284                 totalFiles++;
285             } catch (IOException ex) {
286                 LOGGER.error("Artifact index error", ex);
287             }
288         }
289     }
290 
291     public Indexer getIndexer() {
292         return indexer;
293     }
294 
295     public void setIndexer(Indexer indexer) {
296         this.indexer = indexer;
297     }
298 
299     public Scanner getScanner() {
300         return scanner;
301     }
302 
303     public void setScanner(Scanner scanner) {
304         this.scanner = scanner;
305     }
306 
307     public List<IndexCreator> getIndexers() {
308         return indexers;
309     }
310 
311     public void setIndexers(List<IndexCreator> indexers) {
312         this.indexers = indexers;
313     }
314 
315     public IndexingContext getIndexingContext() {
316         return indexingContext;
317     }
318 
319     public void setIndexingContext(IndexingContext indexingContext) {
320         this.indexingContext = indexingContext;
321     }
322 
323     public String getRepositoryId() {
324         return repositoryId;
325     }
326 
327     public void setRepositoryId(String repositoryId) {
328         this.repositoryId = repositoryId;
329     }
330 
331     public File getRepositoryBasedir() {
332         return repositoryBasedir;
333     }
334 
335     public void setRepositoryBasedir(File repositoryBasedir) {
336         this.repositoryBasedir = repositoryBasedir;
337     }
338 
339     public File getIndexDir() {
340         return indexDir;
341     }
342 
343     public void setIndexDir(File indexDir) {
344         this.indexDir = indexDir;
345     }
346 }