View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.Collections;
25  import java.util.Iterator;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.KeyValue;
32  import org.apache.hadoop.hbase.KeyValue.KVComparator;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.hbase.regionserver.compactions.CompactionConfiguration;
35  
36  import com.google.common.collect.ImmutableCollection;
37  import com.google.common.collect.ImmutableList;
38  import com.google.common.collect.Lists;
39  
40  /**
41   * Default implementation of StoreFileManager. Not thread-safe.
42   */
43  @InterfaceAudience.Private
44  class DefaultStoreFileManager implements StoreFileManager {
45    static final Log LOG = LogFactory.getLog(DefaultStoreFileManager.class);
46  
47    private final KVComparator kvComparator;
48    private final CompactionConfiguration comConf;
49    private final int blockingFileCount;
50  
51    /**
52     * List of store files inside this store. This is an immutable list that
53     * is atomically replaced when its contents change.
54     */
55    private volatile ImmutableList<StoreFile> storefiles = null;
56  
57    public DefaultStoreFileManager(KVComparator kvComparator, Configuration conf,
58        CompactionConfiguration comConf) {
59      this.kvComparator = kvComparator;
60      this.comConf = comConf;
61      this.blockingFileCount =
62          conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT);
63    }
64  
65    @Override
66    public void loadFiles(List<StoreFile> storeFiles) {
67      sortAndSetStoreFiles(storeFiles);
68    }
69  
70    @Override
71    public final Collection<StoreFile> getStorefiles() {
72      return storefiles;
73    }
74  
75    @Override
76    public void insertNewFiles(Collection<StoreFile> sfs) throws IOException {
77      ArrayList<StoreFile> newFiles = new ArrayList<StoreFile>(storefiles);
78      newFiles.addAll(sfs);
79      sortAndSetStoreFiles(newFiles);
80    }
81  
82    @Override
83    public ImmutableCollection<StoreFile> clearFiles() {
84      ImmutableList<StoreFile> result = storefiles;
85      storefiles = ImmutableList.of();
86      return result;
87    }
88  
89    @Override
90    public final int getStorefileCount() {
91      return storefiles.size();
92    }
93  
94    @Override
95    public void addCompactionResults(
96      Collection<StoreFile> compactedFiles, Collection<StoreFile> results) {
97      ArrayList<StoreFile> newStoreFiles = Lists.newArrayList(storefiles);
98      newStoreFiles.removeAll(compactedFiles);
99      if (!results.isEmpty()) {
100       newStoreFiles.addAll(results);
101     }
102     sortAndSetStoreFiles(newStoreFiles);
103   }
104 
105   @Override
106   public final Iterator<StoreFile> getCandidateFilesForRowKeyBefore(final KeyValue targetKey) {
107     return new ArrayList<StoreFile>(Lists.reverse(this.storefiles)).iterator();
108   }
109 
110   @Override
111   public Iterator<StoreFile> updateCandidateFilesForRowKeyBefore(
112       Iterator<StoreFile> candidateFiles, final KeyValue targetKey, final KeyValue candidate) {
113     // Default store has nothing useful to do here.
114     // TODO: move this comment when implementing Level:
115     // Level store can trim the list by range, removing all the files which cannot have
116     // any useful candidates less than "candidate".
117     return candidateFiles;
118   }
119 
120   @Override
121   public final byte[] getSplitPoint() throws IOException {
122     if (this.storefiles.isEmpty()) {
123       return null;
124     }
125     return StoreUtils.getLargestFile(this.storefiles).getFileSplitPoint(this.kvComparator);
126   }
127 
128   @Override
129   public final Collection<StoreFile> getFilesForScanOrGet(boolean isGet,
130       byte[] startRow, byte[] stopRow) {
131     // We cannot provide any useful input and already have the files sorted by seqNum.
132     return getStorefiles();
133   }
134 
135   @Override
136   public int getStoreCompactionPriority() {
137     int priority = blockingFileCount - storefiles.size();
138     return (priority == HStore.PRIORITY_USER) ? priority + 1 : priority;
139   }
140 
141   @Override
142   public Collection<StoreFile> getUnneededFiles(long maxTs, List<StoreFile> filesCompacting) {
143     Collection<StoreFile> expiredStoreFiles = null;
144     ImmutableList<StoreFile> files = storefiles;
145     // 1) We can never get rid of the last file which has the maximum seqid.
146     // 2) Files that are not the latest can't become one due to (1), so the rest are fair game.
147     for (int i = 0; i < files.size() - 1; ++i) {
148       StoreFile sf = files.get(i);
149       long fileTs = sf.getReader().getMaxTimestamp();
150       if (fileTs < maxTs && !filesCompacting.contains(sf)) {
151         LOG.info("Found an expired store file: " + sf.getPath()
152             + " whose maxTimeStamp is " + fileTs + ", which is below " + maxTs);
153         if (expiredStoreFiles == null) {
154           expiredStoreFiles = new ArrayList<StoreFile>();
155         }
156         expiredStoreFiles.add(sf);
157       }
158     }
159     return expiredStoreFiles;
160   }
161 
162   private void sortAndSetStoreFiles(List<StoreFile> storeFiles) {
163     Collections.sort(storeFiles, StoreFile.Comparators.SEQ_ID);
164     storefiles = ImmutableList.copyOf(storeFiles);
165   }
166 
167   @Override
168   public double getCompactionPressure() {
169     int storefileCount = getStorefileCount();
170     int minFilesToCompact = comConf.getMinFilesToCompact();
171     if (storefileCount <= minFilesToCompact) {
172       return 0.0;
173     }
174     return (double) (storefileCount - minFilesToCompact) / (blockingFileCount - minFilesToCompact);
175   }
176 }
177