View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.index;
20  
21  import java.io.File;
22  import java.io.FileInputStream;
23  import java.util.HashSet;
24  import java.util.Properties;
25  import java.util.Set;
26  
27  import org.apache.lucene.search.IndexSearcher;
28  import org.apache.maven.index.context.IndexingContext;
29  import org.apache.maven.index.packer.IndexPacker;
30  import org.apache.maven.index.packer.IndexPackingRequest;
31  import org.codehaus.plexus.util.FileUtils;
32  import org.junit.Test;
33  
34  import static org.junit.Assert.assertEquals;
35  import static org.junit.Assert.assertFalse;
36  import static org.junit.Assert.assertNotNull;
37  import static org.junit.Assert.assertNull;
38  import static org.junit.Assert.assertTrue;
39  
40  // FIXME - hardcoded assumptions in test that break with lucene 4, or bugs?
41  // @Ignore("Segment merge may work differently in Lucene 4")
42  public class Nexus1911IncrementalTest extends AbstractIndexCreatorHelper {
43      NexusIndexer indexer;
44  
45      IndexingContext context;
46  
47      IndexingContext reindexedContext;
48  
49      IndexPacker packer;
50  
51      File indexDir;
52  
53      File indexPackDir;
54  
55      File reposTargetDir;
56  
57      @Override
58      public void setUp() throws Exception {
59          super.setUp();
60  
61          indexer = lookup(NexusIndexer.class);
62          packer = lookup(IndexPacker.class);
63  
64          indexDir = super.getDirectory("index/nexus-1911");
65          indexPackDir = indexDir; // super.getDirectory( "index/nexus-1911-pack" );
66  
67          File reposSrcDir = new File(getBasedir(), "src/test/nexus-1911");
68          this.reposTargetDir = super.getDirectory("repos/nexus-1911");
69  
70          FileUtils.copyDirectoryStructure(reposSrcDir, reposTargetDir);
71  
72          File repo = new File(reposTargetDir, "repo");
73          repo.mkdirs();
74          reindexedContext =
75                  context = indexer.addIndexingContext("test", "test", repo, indexDir, null, null, DEFAULT_CREATORS);
76          indexer.scan(context);
77      }
78  
79      @Override
80      public void tearDown() throws Exception {
81          indexer.removeIndexingContext(context, true);
82          super.deleteDirectory(this.reposTargetDir);
83          super.deleteDirectory(this.indexDir);
84          super.deleteDirectory(this.indexPackDir);
85          super.tearDown();
86      }
87  
88      @Test
89      public void testNoIncremental() throws Exception {
90          final IndexSearcher indexSearcher = context.acquireIndexSearcher();
91          try {
92              IndexPackingRequest request =
93                      new IndexPackingRequest(context, indexSearcher.getIndexReader(), indexPackDir);
94              request.setCreateIncrementalChunks(true);
95              packer.packIndex(request);
96          } finally {
97              context.releaseIndexSearcher(indexSearcher);
98          }
99  
100         Set<String> filenames = getFilenamesFromFiles(indexPackDir.listFiles());
101         Properties props = getPropertiesFromFiles(indexPackDir.listFiles());
102 
103         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".gz"));
104         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".properties"));
105         assertFalse(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".1.gz"));
106         assertFalse(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".2.gz"));
107 
108         assertNotNull(props);
109 
110         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "0"));
111         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "1"));
112         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "2"));
113         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "3"));
114         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "4"));
115         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_COUNTER), "0");
116         assertNotNull(props.getProperty(IndexingContext.INDEX_CHAIN_ID));
117     }
118 
119     @Test
120     public void test1Incremental() throws Exception {
121         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
122         try {
123             IndexPackingRequest request =
124                     new IndexPackingRequest(context, indexSearcher.getIndexReader(), indexPackDir);
125             request.setCreateIncrementalChunks(true);
126             packer.packIndex(request);
127         } finally {
128             context.releaseIndexSearcher(indexSearcher);
129         }
130 
131         copyRepoContentsAndReindex(
132                 new File(getBasedir(), "src/test/nexus-1911/repo-inc-1"), IndexPackingRequest.MAX_CHUNKS);
133 
134         Set<String> filenames = getFilenamesFromFiles(indexPackDir.listFiles());
135         Properties props = getPropertiesFromFiles(indexPackDir.listFiles());
136 
137         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".gz"));
138         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".properties"));
139         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".1.gz"));
140         assertFalse(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".2.gz"));
141 
142         assertNotNull(props);
143 
144         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "0"), "1");
145         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "1"));
146         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "2"));
147         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "3"));
148         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "4"));
149         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_COUNTER), "1");
150         assertNotNull(props.getProperty(IndexingContext.INDEX_CHAIN_ID));
151     }
152 
153     @Test
154     public void test2Incremental() throws Exception {
155         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
156         try {
157             IndexPackingRequest request =
158                     new IndexPackingRequest(context, indexSearcher.getIndexReader(), indexPackDir);
159             request.setCreateIncrementalChunks(true);
160             packer.packIndex(request);
161         } finally {
162             context.releaseIndexSearcher(indexSearcher);
163         }
164 
165         copyRepoContentsAndReindex(
166                 new File(getBasedir(), "src/test/nexus-1911/repo-inc-1"), IndexPackingRequest.MAX_CHUNKS);
167         copyRepoContentsAndReindex(
168                 new File(getBasedir(), "src/test/nexus-1911/repo-inc-2"), IndexPackingRequest.MAX_CHUNKS);
169 
170         Set<String> filenames = getFilenamesFromFiles(indexPackDir.listFiles());
171         Properties props = getPropertiesFromFiles(indexPackDir.listFiles());
172 
173         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".gz"));
174         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".properties"));
175         // 1 is missing with updated Lucene 4 implementation
176         //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
177         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".2.gz"));
178         assertFalse(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".3.gz"));
179 
180         assertNotNull(props);
181 
182         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "0"), "2");
183         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "1"), "1");
184         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "2"));
185         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "3"));
186         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "4"));
187         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_COUNTER), "2");
188         assertNotNull(props.getProperty(IndexingContext.INDEX_CHAIN_ID));
189     }
190 
191     @Test
192     public void test3Incremental() throws Exception {
193         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
194         try {
195             IndexPackingRequest request =
196                     new IndexPackingRequest(context, indexSearcher.getIndexReader(), indexPackDir);
197             request.setCreateIncrementalChunks(true);
198             packer.packIndex(request);
199         } finally {
200             context.releaseIndexSearcher(indexSearcher);
201         }
202 
203         copyRepoContentsAndReindex(
204                 new File(getBasedir(), "src/test/nexus-1911/repo-inc-1"), IndexPackingRequest.MAX_CHUNKS);
205         copyRepoContentsAndReindex(
206                 new File(getBasedir(), "src/test/nexus-1911/repo-inc-2"), IndexPackingRequest.MAX_CHUNKS);
207         copyRepoContentsAndReindex(
208                 new File(getBasedir(), "src/test/nexus-1911/repo-inc-3"), IndexPackingRequest.MAX_CHUNKS);
209 
210         Set<String> filenames = getFilenamesFromFiles(indexPackDir.listFiles());
211         Properties props = getPropertiesFromFiles(indexPackDir.listFiles());
212 
213         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".gz"));
214         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".properties"));
215         // 1,2 are missing with updated Lucene 4 implementation
216         //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".1.gz" ) );
217         //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
218         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".3.gz"));
219 
220         assertNotNull(props);
221 
222         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "0"), "3");
223         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "1"), "2");
224         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "2"), "1");
225         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "3"));
226         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "4"));
227         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_COUNTER), "3");
228         assertNotNull(props.getProperty(IndexingContext.INDEX_CHAIN_ID));
229     }
230 
231     @Test
232     public void testMaxChunks() throws Exception {
233         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
234         try {
235             IndexPackingRequest request =
236                     new IndexPackingRequest(context, indexSearcher.getIndexReader(), indexPackDir);
237             request.setCreateIncrementalChunks(true);
238             request.setMaxIndexChunks(3);
239             packer.packIndex(request);
240         } finally {
241             context.releaseIndexSearcher(indexSearcher);
242         }
243 
244         copyRepoContentsAndReindex(new File(getBasedir(), "src/test/nexus-1911/repo-inc-1"), 3);
245         copyRepoContentsAndReindex(new File(getBasedir(), "src/test/nexus-1911/repo-inc-2"), 3);
246         copyRepoContentsAndReindex(new File(getBasedir(), "src/test/nexus-1911/repo-inc-3"), 3);
247         copyRepoContentsAndReindex(new File(getBasedir(), "src/test/nexus-1911/repo-inc-4"), 3);
248 
249         Set<String> filenames = getFilenamesFromFiles(indexPackDir.listFiles());
250         Properties props = getPropertiesFromFiles(indexPackDir.listFiles());
251 
252         System.out.println(filenames);
253 
254         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".gz"));
255         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".properties"));
256         assertFalse(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".1.gz"));
257 
258         // 2,3 are missing with updated Lucene 4 implementation
259         //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".2.gz" ) );
260         //        Assert.assertTrue( filenames.contains( IndexingContext.INDEX_FILE_PREFIX + ".3.gz" ) );
261         assertTrue(filenames.contains(IndexingContext.INDEX_FILE_PREFIX + ".4.gz"));
262 
263         assertNotNull(props);
264 
265         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "0"), "4");
266         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "1"), "3");
267         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "2"), "2");
268         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "3"));
269         assertNull(props.getProperty(IndexingContext.INDEX_CHUNK_PREFIX + "4"));
270         assertEquals(props.getProperty(IndexingContext.INDEX_CHUNK_COUNTER), "4");
271         assertNotNull(props.getProperty(IndexingContext.INDEX_CHAIN_ID));
272     }
273 
274     private void copyRepoContentsAndReindex(File src, int maxIndexChunks) throws Exception {
275         File reposTargetDir = new File(getBasedir(), "target/repos/nexus-1911/repo");
276 
277         FileUtils.copyDirectoryStructure(src, reposTargetDir);
278 
279         // this was ALWAYS broken, if incremental reindex wanted, this has to be TRUE!!!
280         // TODO: fix this!
281         indexer.scan(reindexedContext, false);
282 
283         final IndexSearcher indexSearcher = context.acquireIndexSearcher();
284         try {
285             IndexPackingRequest request =
286                     new IndexPackingRequest(context, indexSearcher.getIndexReader(), indexPackDir);
287             request.setCreateIncrementalChunks(true);
288             request.setMaxIndexChunks(maxIndexChunks);
289             packer.packIndex(request);
290         } finally {
291             context.releaseIndexSearcher(indexSearcher);
292         }
293     }
294 
295     private Set<String> getFilenamesFromFiles(File[] files) {
296         Set<String> filenames = new HashSet<>();
297 
298         for (File file : files) {
299             filenames.add(file.getName());
300         }
301 
302         return filenames;
303     }
304 
305     private Properties getPropertiesFromFiles(File[] files) throws Exception {
306         Properties props = new Properties();
307         File propertyFile = null;
308 
309         for (File file : files) {
310             if ((IndexingContext.INDEX_REMOTE_PROPERTIES_FILE).equalsIgnoreCase(file.getName())) {
311                 propertyFile = file;
312                 break;
313             }
314         }
315 
316         try (FileInputStream fis = new FileInputStream(propertyFile)) {
317             props.load(fis);
318         }
319 
320         return props;
321     }
322 }