View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.index.reader;
20  
21  import java.io.Closeable;
22  import java.io.IOException;
23  import java.io.UncheckedIOException;
24  import java.text.ParseException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.Date;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Properties;
31  import java.util.concurrent.atomic.AtomicBoolean;
32  
33  import org.apache.maven.index.reader.ResourceHandler.Resource;
34  
35  import static java.util.Objects.requireNonNull;
36  import static org.apache.maven.index.reader.Utils.loadProperties;
37  import static org.apache.maven.index.reader.Utils.storeProperties;
38  
39  /**
40   * Maven Index reader that handles incremental updates, if possible, and provides one or more {@link ChunkReader}s, to
41   * read all the required records. Instances of this class MUST BE handled as resources (have them closed once done with
42   * them), it is user responsibility to close them, ideally in try-with-resource block.
43   * <p>
44   * Every involved instance, this {@link IndexReader}, provided {@link ChunkReader}, and used  {@link ResourceHandler}s
45   * are {@link Closeable}, and all have to be explicitly closed, best in try-with-resource.
46   *
47   * @since 5.1.2
48   */
49  public class IndexReader implements Iterable<ChunkReader>, Closeable {
50      private final AtomicBoolean closed;
51  
52      private final WritableResourceHandler local;
53  
54      private final ResourceHandler remote;
55  
56      private final Properties localIndexProperties;
57  
58      private final Properties remoteIndexProperties;
59  
60      private final String indexId;
61  
62      private final Date publishedTimestamp;
63  
64      private final boolean incremental;
65  
66      private final List<String> chunkNames;
67  
68      public IndexReader(final WritableResourceHandler local, final ResourceHandler remote) throws IOException {
69          requireNonNull(remote, "remote resource handler null");
70          this.closed = new AtomicBoolean(false);
71          this.local = local;
72          this.remote = remote;
73          remoteIndexProperties = loadProperties(remote.locate(Utils.INDEX_FILE_PREFIX + ".properties"));
74          if (remoteIndexProperties == null) {
75              throw new IllegalArgumentException("Non-existent remote index");
76          }
77          try {
78              if (local != null) {
79                  Properties localProperties = loadProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"));
80                  if (localProperties != null) {
81                      this.localIndexProperties = localProperties;
82                      String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id");
83                      String localIndexId = localIndexProperties.getProperty("nexus.index.id");
84                      if (remoteIndexId == null || !remoteIndexId.equals(localIndexId)) {
85                          throw new IllegalArgumentException("local and remote index IDs does not match or is null: "
86                                  + localIndexId + ", " + remoteIndexId);
87                      }
88                      this.indexId = localIndexId;
89                      this.incremental = canRetrieveAllChunks();
90                  } else {
91                      localIndexProperties = null;
92                      this.indexId = remoteIndexProperties.getProperty("nexus.index.id");
93                      this.incremental = false;
94                  }
95              } else {
96                  localIndexProperties = null;
97                  this.indexId = remoteIndexProperties.getProperty("nexus.index.id");
98                  this.incremental = false;
99              }
100             this.publishedTimestamp =
101                     Utils.INDEX_DATE_FORMAT.parse(remoteIndexProperties.getProperty("nexus.index.timestamp"));
102             this.chunkNames = calculateChunkNames();
103         } catch (ParseException e) {
104             throw new IOException("Index properties corrupted", e);
105         }
106     }
107 
108     /**
109      * Returns the index context ID that published index has set. Usually it is equal to "repository ID" used in {@link
110      * Record.Type#DESCRIPTOR} but does not have to be.
111      */
112     public String getIndexId() {
113         return indexId;
114     }
115 
116     /**
117      * Returns the {@link Date} when remote index was last published.
118      */
119     public Date getPublishedTimestamp() {
120         return publishedTimestamp;
121     }
122 
123     /**
124      * Returns {@code true} if incremental update is about to happen. If incremental update, the {@link #iterator()}
125      * will return only the diff from the last update.
126      */
127     public boolean isIncremental() {
128         return incremental;
129     }
130 
131     /**
132      * Returns unmodifiable list of actual chunks that needs to be pulled from remote {@link ResourceHandler}. Those are
133      * incremental chunks or the big main file, depending on result of {@link #isIncremental()}. Empty list means local
134      * index is up to date, and {@link #iterator()} will return empty iterator.
135      */
136     public List<String> getChunkNames() {
137         return chunkNames;
138     }
139 
140     /**
141      * Closes the underlying {@link ResourceHandler}s. In case of incremental update use, it also assumes that user
142      * consumed all the iterator and integrated it, hence, it will update the {@link WritableResourceHandler} contents
143      * to prepare it for future incremental update. If this is not desired (ie. due to aborted update), then this
144      * method should NOT be invoked, but rather the {@link ResourceHandler}s that caller provided in constructor of
145      * this class should be closed manually.
146      */
147     @Override
148     public void close() throws IOException {
149         if (closed.compareAndSet(false, true)) {
150             remote.close();
151             if (local != null) {
152                 try {
153                     syncLocalWithRemote();
154                 } finally {
155                     local.close();
156                 }
157             }
158         }
159     }
160 
161     /**
162      * Returns an {@link Iterator} of {@link ChunkReader}s, that if read in sequence, provide all the (incremental)
163      * updates from the index. It is caller responsibility to either consume fully this iterator, or to close current
164      * {@link ChunkReader} if aborting.
165      */
166     @Override
167     public Iterator<ChunkReader> iterator() {
168         return new ChunkReaderIterator(remote, chunkNames.iterator());
169     }
170 
171     /**
172      * Stores the remote index properties into local index properties, preparing local {@link WritableResourceHandler}
173      * for future incremental updates.
174      */
175     private void syncLocalWithRemote() throws IOException {
176         storeProperties(local.locate(Utils.INDEX_FILE_PREFIX + ".properties"), remoteIndexProperties);
177     }
178 
179     /**
180      * Calculates the chunk names that needs to be fetched.
181      */
182     private List<String> calculateChunkNames() {
183         if (incremental) {
184             ArrayList<String> chunkNames = new ArrayList<>();
185             int maxCounter = Integer.parseInt(remoteIndexProperties.getProperty("nexus.index.last-incremental"));
186             int currentCounter = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental"));
187             currentCounter++;
188             while (currentCounter <= maxCounter) {
189                 chunkNames.add(Utils.INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz");
190             }
191             return Collections.unmodifiableList(chunkNames);
192         } else {
193             return Collections.singletonList(Utils.INDEX_FILE_PREFIX + ".gz");
194         }
195     }
196 
197     /**
198      * Verifies incremental update is possible, as all the diff chunks we need are still enlisted in remote properties.
199      */
200     private boolean canRetrieveAllChunks() {
201         String localChainId = localIndexProperties.getProperty("nexus.index.chain-id");
202         String remoteChainId = remoteIndexProperties.getProperty("nexus.index.chain-id");
203 
204         // If no chain id, or not the same, do full update
205         if (localChainId == null || !localChainId.equals(remoteChainId)) {
206             return false;
207         }
208 
209         try {
210             int localLastIncremental =
211                     Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental"));
212             String currentLocalCounter = String.valueOf(localLastIncremental);
213             String nextLocalCounter = String.valueOf(localLastIncremental + 1);
214             // check remote props for existence of current or next chunk after local
215             for (Object key : remoteIndexProperties.keySet()) {
216                 String sKey = (String) key;
217                 if (sKey.startsWith("nexus.index.incremental-")) {
218                     String value = remoteIndexProperties.getProperty(sKey);
219                     if (currentLocalCounter.equals(value) || nextLocalCounter.equals(value)) {
220                         return true;
221                     }
222                 }
223             }
224         } catch (NumberFormatException e) {
225             // fall through
226         }
227         return false;
228     }
229 
230     /**
231      * Internal iterator implementation that lazily opens and closes the returned {@link ChunkReader}s as this iterator
232      * is being consumed.
233      */
234     private static class ChunkReaderIterator implements Iterator<ChunkReader> {
235         private final ResourceHandler resourceHandler;
236 
237         private final Iterator<String> chunkNamesIterator;
238 
239         private ChunkReaderIterator(final ResourceHandler resourceHandler, final Iterator<String> chunkNamesIterator) {
240             this.resourceHandler = resourceHandler;
241             this.chunkNamesIterator = chunkNamesIterator;
242         }
243 
244         @Override
245         public boolean hasNext() {
246             return chunkNamesIterator.hasNext();
247         }
248 
249         @Override
250         public ChunkReader next() {
251             String chunkName = chunkNamesIterator.next();
252             try {
253                 Resource currentResource = resourceHandler.locate(chunkName);
254                 return new ChunkReader(chunkName, currentResource.read());
255             } catch (IOException e) {
256                 throw new UncheckedIOException("IO problem while opening chunk readers", e);
257             }
258         }
259 
260         @Override
261         public void remove() {
262             throw new UnsupportedOperationException("remove");
263         }
264     }
265 }