@InterfaceAudience.Public @InterfaceStability.Stable public class LoadIncrementalHFiles extends Configured
usage()
Modifier and Type | Class and Description |
---|---|
static class |
LoadIncrementalHFiles.LoadQueueItem
Represents an HFile waiting to be loaded.
|
Modifier and Type | Field and Description |
---|---|
static java.lang.String |
ALWAYS_COPY_FILES |
static java.lang.String |
CREATE_TABLE_CONF_KEY |
static java.lang.String |
MAX_FILES_PER_REGION_PER_FAMILY |
static java.lang.String |
NAME |
static java.lang.String |
SILENCE_CONF_KEY |
Constructor and Description |
---|
LoadIncrementalHFiles(Configuration conf) |
Modifier and Type | Method and Description |
---|---|
protected void |
bulkLoadPhase(Table table,
Connection conn,
java.util.concurrent.ExecutorService pool,
java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue,
<any> regionGroups,
boolean copyFile,
java.util.Map<LoadIncrementalHFiles.LoadQueueItem,java.nio.ByteBuffer> item2RegionMap)
This takes the LQI's grouped by likely regions and attempts to bulk load
them.
|
void |
doBulkLoad(java.util.Map<byte[],java.util.List<Path>> map,
Admin admin,
Table table,
RegionLocator regionLocator,
boolean silence,
boolean copyFile)
Perform a bulk load of the given directory into the given
pre-existing table.
|
void |
doBulkLoad(Path hfofDir,
Admin admin,
Table table,
RegionLocator regionLocator)
Perform a bulk load of the given directory into the given
pre-existing table.
|
void |
doBulkLoad(Path hfofDir,
Admin admin,
Table table,
RegionLocator regionLocator,
boolean silence,
boolean copyFile)
Perform a bulk load of the given directory into the given
pre-existing table.
|
void |
doBulkLoad(Path hfofDir,
HTable table)
Perform a bulk load of the given directory into the given
pre-existing table.
|
protected Pair<java.util.List<LoadIncrementalHFiles.LoadQueueItem>,java.lang.String> |
groupOrSplit(<any> regionGroups,
LoadIncrementalHFiles.LoadQueueItem item,
Table table,
Pair<byte[][],byte[][]> startEndKeys)
Attempt to assign the given load queue item into its target region group.
|
static byte[][] |
inferBoundaries(java.util.TreeMap<byte[],java.lang.Integer> bdryMap) |
void |
loadHFileQueue(Table table,
Connection conn,
java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue,
Pair<byte[][],byte[][]> startEndKeys)
Used by the replication sink to load the hfiles from the source cluster.
|
void |
loadHFileQueue(Table table,
Connection conn,
java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue,
Pair<byte[][],byte[][]> startEndKeys,
boolean copyFile)
Used by the replication sink to load the hfiles from the source cluster.
|
static void |
main(java.lang.String[] args) |
void |
prepareHFileQueue(java.util.Map<byte[],java.util.List<Path>> map,
Table table,
java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue,
boolean silence)
Prepare a collection of
LoadIncrementalHFiles.LoadQueueItem from list of source hfiles contained in the
passed directory and validates whether the prepared queue has all the valid table column
families in it. |
void |
prepareHFileQueue(Path hfofDir,
Table table,
java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue,
boolean validateHFile)
Prepare a collection of
LoadIncrementalHFiles.LoadQueueItem from list of source hfiles contained in the
passed directory and validates whether the prepared queue has all the valid table column
families in it. |
void |
prepareHFileQueue(Path hfilesDir,
Table table,
java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue,
boolean validateHFile,
boolean silence)
Prepare a collection of
LoadIncrementalHFiles.LoadQueueItem from list of source hfiles contained in the
passed directory and validates whether the prepared queue has all the valid table column
families in it. |
int |
run(java.lang.String[] args) |
java.util.Map<LoadIncrementalHFiles.LoadQueueItem,java.nio.ByteBuffer> |
run(java.lang.String dirPath,
java.util.Map<byte[],java.util.List<Path>> map,
TableName tableName) |
void |
setBulkToken(java.lang.String stagingDir)
Called from replication sink, where it manages bulkToken(staging directory) by itself.
|
protected java.util.List<LoadIncrementalHFiles.LoadQueueItem> |
splitStoreFile(LoadIncrementalHFiles.LoadQueueItem item,
Table table,
byte[] startKey,
byte[] splitKey) |
protected java.util.List<LoadIncrementalHFiles.LoadQueueItem> |
tryAtomicRegionLoad(Connection conn,
TableName tableName,
byte[] first,
java.util.Collection<LoadIncrementalHFiles.LoadQueueItem> lqis,
boolean copyFile)
Attempts to do an atomic load of many hfiles into a region.
|
protected java.util.List<LoadIncrementalHFiles.LoadQueueItem> |
tryAtomicRegionLoad(HConnection conn,
byte[] tableName,
byte[] first,
java.util.Collection<LoadIncrementalHFiles.LoadQueueItem> lqis)
Deprecated.
As of release 0.96
(HBASE-9508).
This will be removed in HBase 2.0.0.
Use
#tryAtomicRegionLoad(Connection, TableName, byte[], Collection) . |
public static final java.lang.String NAME
public static final java.lang.String MAX_FILES_PER_REGION_PER_FAMILY
public static final java.lang.String CREATE_TABLE_CONF_KEY
public static final java.lang.String SILENCE_CONF_KEY
public static final java.lang.String ALWAYS_COPY_FILES
public LoadIncrementalHFiles(Configuration conf) throws java.lang.Exception
java.lang.Exception
public void doBulkLoad(Path hfofDir, HTable table) throws TableNotFoundException, java.io.IOException
hfofDir
- the directory that was provided as the output path
of a job using HFileOutputFormattable
- the table to load intoTableNotFoundException
- if table does not yet existjava.io.IOException
public void doBulkLoad(Path hfofDir, Admin admin, Table table, RegionLocator regionLocator) throws TableNotFoundException, java.io.IOException
hfofDir
- the directory that was provided as the output path
of a job using HFileOutputFormatadmin
- the Admintable
- the table to load intoregionLocator
- region locatorTableNotFoundException
- if table does not yet existjava.io.IOException
public void doBulkLoad(java.util.Map<byte[],java.util.List<Path>> map, Admin admin, Table table, RegionLocator regionLocator, boolean silence, boolean copyFile) throws TableNotFoundException, java.io.IOException
map
- map of family to List of hfilesadmin
- the Admintable
- the table to load intoregionLocator
- region locatorsilence
- true to ignore unmatched column familiescopyFile
- always copy hfiles if trueTableNotFoundException
- if table does not yet existjava.io.IOException
public void doBulkLoad(Path hfofDir, Admin admin, Table table, RegionLocator regionLocator, boolean silence, boolean copyFile) throws TableNotFoundException, java.io.IOException
hfofDir
- the directory that was provided as the output path
of a job using HFileOutputFormatadmin
- the Admintable
- the table to load intoregionLocator
- region locatorsilence
- true to ignore unmatched column familiescopyFile
- always copy hfiles if trueTableNotFoundException
- if table does not yet existjava.io.IOException
public void prepareHFileQueue(Path hfofDir, Table table, java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue, boolean validateHFile) throws java.io.IOException
LoadIncrementalHFiles.LoadQueueItem
from list of source hfiles contained in the
passed directory and validates whether the prepared queue has all the valid table column
families in it.hfilesDir
- directory containing list of hfiles to be loaded into the tabletable
- table to which hfiles should be loadedqueue
- queue which needs to be loaded into the tablevalidateHFile
- if true hfiles will be validated for its formatjava.io.IOException
- If any I/O or network error occurredpublic void prepareHFileQueue(Path hfilesDir, Table table, java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue, boolean validateHFile, boolean silence) throws java.io.IOException
LoadIncrementalHFiles.LoadQueueItem
from list of source hfiles contained in the
passed directory and validates whether the prepared queue has all the valid table column
families in it.hfilesDir
- directory containing list of hfiles to be loaded into the tabletable
- table to which hfiles should be loadedqueue
- queue which needs to be loaded into the tablevalidateHFile
- if true hfiles will be validated for its formatsilence
- true to ignore unmatched column familiesjava.io.IOException
- If any I/O or network error occurredpublic void prepareHFileQueue(java.util.Map<byte[],java.util.List<Path>> map, Table table, java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue, boolean silence) throws java.io.IOException
LoadIncrementalHFiles.LoadQueueItem
from list of source hfiles contained in the
passed directory and validates whether the prepared queue has all the valid table column
families in it.map
- map of family to List of hfilestable
- table to which hfiles should be loadedqueue
- queue which needs to be loaded into the tablesilence
- true to ignore unmatched column familiesjava.io.IOException
- If any I/O or network error occurredpublic void loadHFileQueue(Table table, Connection conn, java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue, Pair<byte[][],byte[][]> startEndKeys) throws java.io.IOException
groupOrSplitPhase(Table, ExecutorService, Deque, Pair)
2.
LoadIncrementalHFiles#bulkLoadPhase(Table, Connection, ExecutorService, Deque, Multimap)
table
- Table to which these hfiles should be loaded toconn
- Connection to usequeue
- LoadIncrementalHFiles.LoadQueueItem
has hfiles yet to be loadedstartEndKeys
- starting and ending row keys of the regionjava.io.IOException
public void loadHFileQueue(Table table, Connection conn, java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue, Pair<byte[][],byte[][]> startEndKeys, boolean copyFile) throws java.io.IOException
table
- Table to which these hfiles should be loaded toconn
- Connection to usequeue
- LoadIncrementalHFiles.LoadQueueItem
has hfiles yet to be loadedstartEndKeys
- starting and ending row keys of the regionjava.io.IOException
protected void bulkLoadPhase(Table table, Connection conn, java.util.concurrent.ExecutorService pool, java.util.Deque<LoadIncrementalHFiles.LoadQueueItem> queue, <any> regionGroups, boolean copyFile, java.util.Map<LoadIncrementalHFiles.LoadQueueItem,java.nio.ByteBuffer> item2RegionMap) throws java.io.IOException
java.io.IOException
protected java.util.List<LoadIncrementalHFiles.LoadQueueItem> splitStoreFile(LoadIncrementalHFiles.LoadQueueItem item, Table table, byte[] startKey, byte[] splitKey) throws java.io.IOException
java.io.IOException
protected Pair<java.util.List<LoadIncrementalHFiles.LoadQueueItem>,java.lang.String> groupOrSplit(<any> regionGroups, LoadIncrementalHFiles.LoadQueueItem item, Table table, Pair<byte[][],byte[][]> startEndKeys) throws java.io.IOException
java.io.IOException
@Deprecated protected java.util.List<LoadIncrementalHFiles.LoadQueueItem> tryAtomicRegionLoad(HConnection conn, byte[] tableName, byte[] first, java.util.Collection<LoadIncrementalHFiles.LoadQueueItem> lqis) throws java.io.IOException
#tryAtomicRegionLoad(Connection, TableName, byte[], Collection)
.java.io.IOException
protected java.util.List<LoadIncrementalHFiles.LoadQueueItem> tryAtomicRegionLoad(Connection conn, TableName tableName, byte[] first, java.util.Collection<LoadIncrementalHFiles.LoadQueueItem> lqis, boolean copyFile) throws java.io.IOException
java.io.IOException
public static byte[][] inferBoundaries(java.util.TreeMap<byte[],java.lang.Integer> bdryMap)
public java.util.Map<LoadIncrementalHFiles.LoadQueueItem,java.nio.ByteBuffer> run(java.lang.String dirPath, java.util.Map<byte[],java.util.List<Path>> map, TableName tableName) throws java.io.IOException
java.io.IOException
public int run(java.lang.String[] args) throws java.lang.Exception
java.lang.Exception
public static void main(java.lang.String[] args) throws java.lang.Exception
java.lang.Exception
public void setBulkToken(java.lang.String stagingDir)
SecureBulkLoadEndpoint
is configured in hbase.coprocessor.region.classes
property. This directory is used as a temporary directory where all files are initially
copied/moved from user given directory, set all the required file permissions and then from
their it is finally loaded into a table. This should be set only when, one would like to manage
the staging directory by itself. Otherwise this tool will handle this by itself.stagingDir
- staging directory path