public class SnapPuller
extends java.lang.Object
Modifier and Type | Class and Description |
---|---|
private class |
SnapPuller.FileFetcher
The class acts as a client for ReplicationHandler.FileStream.
|
private static class |
SnapPuller.ReplicationHandlerException |
Modifier and Type | Field and Description |
---|---|
private static org.apache.commons.httpclient.HttpClient |
client |
(package private) static java.lang.String |
CONF_FILES_REPLICATED |
(package private) static java.lang.String |
CONF_FILES_REPLICATED_AT |
private java.util.Map<java.lang.String,ReplicationHandler.FileInfo> |
confFileInfoCache |
private java.util.List<java.util.Map<java.lang.String,java.lang.Object>> |
confFilesDownloaded |
private java.util.List<java.util.Map<java.lang.String,java.lang.Object>> |
confFilesToDownload |
private java.util.Map<java.lang.String,java.lang.Object> |
currentFile |
private static int |
ERR |
private java.util.concurrent.ScheduledExecutorService |
executorService |
private long |
executorStartTime |
private SnapPuller.FileFetcher |
fileFetcher |
private java.util.List<java.util.Map<java.lang.String,java.lang.Object>> |
filesDownloaded |
private java.util.List<java.util.Map<java.lang.String,java.lang.Object>> |
filesToDownload |
private java.lang.Exception |
fsyncException |
private java.util.concurrent.ExecutorService |
fsyncService |
private static java.lang.String |
HTTP_BASIC_AUTH_PASSWORD |
private static java.lang.String |
HTTP_BASIC_AUTH_USER |
private static java.lang.String |
HTTP_CONN_TIMEOUT |
private static java.lang.String |
HTTP_READ_TIMEOUT |
(package private) static java.lang.String |
INDEX_REPLICATED_AT |
(package private) static java.lang.String |
INDEX_REPLICATED_AT_LIST |
static java.lang.String |
INTERVAL_ERR_MSG |
private static java.util.regex.Pattern |
INTERVAL_PATTERN |
(package private) static java.lang.String |
LAST_CYCLE_BYTES_DOWNLOADED |
private static org.slf4j.Logger |
LOG |
private java.lang.String |
masterUrl |
private static int |
MAX_RETRIES |
private org.apache.commons.httpclient.HttpClient |
myHttpClient |
private static int |
NO_CONTENT |
static java.lang.String |
POLL_INTERVAL |
private java.util.concurrent.atomic.AtomicBoolean |
pollDisabled
Disable the timer task for polling
|
private java.lang.Integer |
pollInterval |
private java.lang.String |
pollIntervalStr |
(package private) static java.lang.String |
PREVIOUS_CYCLE_TIME_TAKEN |
(package private) static java.lang.String |
REPLICATION_FAILED_AT |
(package private) static java.lang.String |
REPLICATION_FAILED_AT_LIST |
static java.lang.String |
REPLICATION_PROPERTIES |
private ReplicationHandler |
replicationHandler |
private long |
replicationStartTime |
private SolrCore |
solrCore |
private boolean |
stop |
(package private) boolean |
successfulInstall
This command downloads all the necessary files from master to install a index commit point.
|
(package private) static java.lang.String |
TIMES_CONFIG_REPLICATED |
(package private) static java.lang.String |
TIMES_FAILED |
(package private) static java.lang.String |
TIMES_INDEX_REPLICATED |
private boolean |
useExternal |
private boolean |
useInternal |
Constructor and Description |
---|
SnapPuller(NamedList initArgs,
ReplicationHandler handler,
SolrCore sc) |
Modifier and Type | Method and Description |
---|---|
(package private) void |
abortPull()
Stops the ongoing pull
|
private java.io.InputStream |
checkCompressed(org.apache.commons.httpclient.HttpMethod method,
java.io.InputStream respBody) |
private boolean |
copyAFile(java.io.File tmpIdxDir,
java.io.File indexDir,
java.lang.String fname,
java.util.List<java.lang.String> copiedfiles)
Copy a file by the File#renameTo() method.
|
private boolean |
copyIndexFiles(java.io.File tmpIdxDir,
java.io.File indexDir)
Copy all index files from the temp index dir to the actual index.
|
private void |
copyTmpConfFiles2Conf(java.io.File tmpconfDir)
The conf files are copied to the tmp dir to the conf dir.
|
private static org.apache.commons.httpclient.HttpClient |
createHttpClient(java.lang.String connTimeout,
java.lang.String readTimeout) |
private java.io.File |
createTempindexDir(SolrCore core)
All the files are copied to a temp dir first
|
(package private) static boolean |
delTree(java.io.File dir)
Delete the directory tree recursively
|
void |
destroy() |
(package private) void |
disablePoll()
Disable periodic polling
|
private void |
doCommit() |
private void |
downloadConfFiles(java.util.List<java.util.Map<java.lang.String,java.lang.Object>> confFilesToDownload,
long latestVersion) |
private void |
downloadIndexFiles(boolean downloadCompleteIndex,
java.io.File tmpIdxDir,
long latestVersion)
Download the index files.
|
(package private) void |
enablePoll()
Enable periodic polling
|
(package private) void |
fetchFileList(long version)
Fetches the list of files in a given index commit point
|
(package private) boolean |
fetchLatestIndex(SolrCore core) |
(package private) NamedList |
getCommandResponse(NamedList<java.lang.String> commands) |
(package private) java.util.List<java.util.Map<java.lang.String,java.lang.Object>> |
getConfFilesDownloaded() |
(package private) java.util.List<java.util.Map<java.lang.String,java.lang.Object>> |
getConfFilesToDownload() |
(package private) java.util.Map<java.lang.String,java.lang.Object> |
getCurrentFile() |
private java.lang.String |
getDateAsStr(java.util.Date d) |
(package private) java.util.List<java.util.Map<java.lang.String,java.lang.Object>> |
getFilesDownloaded() |
(package private) java.util.List<java.util.Map<java.lang.String,java.lang.Object>> |
getFilesToDownload() |
(package private) NamedList |
getLatestVersion()
Gets the latest commit version and generation from the master
|
(package private) java.lang.String |
getMasterUrl() |
private java.util.Collection<java.util.Map<java.lang.String,java.lang.Object>> |
getModifiedConfFiles(java.util.List<java.util.Map<java.lang.String,java.lang.Object>> confFilesToDownload)
The local conf files are compared with the conf files in the master.
|
private NamedList |
getNamedListResponse(org.apache.commons.httpclient.methods.PostMethod method) |
(package private) java.lang.Long |
getNextScheduledExecTime() |
(package private) java.lang.String |
getPollInterval() |
(package private) long |
getReplicationStartTime() |
(package private) static long |
getTotalBytesDownloaded(SnapPuller snappuller) |
private boolean |
isIndexStale()
All the files which are common between master and slave must have same timestamp and size else we assume they are
not compatible (stale).
|
(package private) boolean |
isPollingDisabled() |
private void |
logReplicationTimeAndConfFiles(java.util.Collection<java.util.Map<java.lang.String,java.lang.Object>> modifiedConfFiles,
boolean successfulInstall)
Helper method to record the last replication's details so that we can show them on the statistics page across
restarts.
|
private boolean |
modifyIndexProps(java.lang.String tmpIdxDirName)
If the index is stale by any chance, load index from a different dir in the data dir.
|
(package private) static java.lang.Integer |
readInterval(java.lang.String interval) |
private java.lang.StringBuffer |
readToStringBuffer(long replicationTime,
java.lang.String str) |
private void |
reloadCore() |
private void |
startExecutorService() |
private void |
terminateAndWaitFsyncService()
terminate the fsync service and wait for all the tasks to complete.
|
private static final org.slf4j.Logger LOG
private final java.lang.String masterUrl
private final ReplicationHandler replicationHandler
private final java.lang.Integer pollInterval
private java.lang.String pollIntervalStr
private java.util.concurrent.ScheduledExecutorService executorService
private volatile long executorStartTime
private volatile long replicationStartTime
private final SolrCore solrCore
private volatile java.util.List<java.util.Map<java.lang.String,java.lang.Object>> filesToDownload
private volatile java.util.List<java.util.Map<java.lang.String,java.lang.Object>> confFilesToDownload
private volatile java.util.List<java.util.Map<java.lang.String,java.lang.Object>> filesDownloaded
private volatile java.util.List<java.util.Map<java.lang.String,java.lang.Object>> confFilesDownloaded
private volatile java.util.Map<java.lang.String,java.lang.Object> currentFile
private volatile SnapPuller.FileFetcher fileFetcher
private volatile java.util.concurrent.ExecutorService fsyncService
private volatile boolean stop
private boolean useInternal
private boolean useExternal
private java.util.concurrent.atomic.AtomicBoolean pollDisabled
private static org.apache.commons.httpclient.HttpClient client
private final org.apache.commons.httpclient.HttpClient myHttpClient
boolean successfulInstall
private volatile java.lang.Exception fsyncException
private final java.util.Map<java.lang.String,ReplicationHandler.FileInfo> confFileInfoCache
private static final int MAX_RETRIES
private static final int NO_CONTENT
private static final int ERR
public static final java.lang.String REPLICATION_PROPERTIES
public static final java.lang.String POLL_INTERVAL
public static final java.lang.String INTERVAL_ERR_MSG
private static final java.util.regex.Pattern INTERVAL_PATTERN
private static final java.lang.String HTTP_CONN_TIMEOUT
private static final java.lang.String HTTP_READ_TIMEOUT
private static final java.lang.String HTTP_BASIC_AUTH_USER
private static final java.lang.String HTTP_BASIC_AUTH_PASSWORD
static final java.lang.String INDEX_REPLICATED_AT
static final java.lang.String TIMES_INDEX_REPLICATED
static final java.lang.String CONF_FILES_REPLICATED
static final java.lang.String CONF_FILES_REPLICATED_AT
static final java.lang.String TIMES_CONFIG_REPLICATED
static final java.lang.String LAST_CYCLE_BYTES_DOWNLOADED
static final java.lang.String TIMES_FAILED
static final java.lang.String REPLICATION_FAILED_AT
static final java.lang.String PREVIOUS_CYCLE_TIME_TAKEN
static final java.lang.String INDEX_REPLICATED_AT_LIST
static final java.lang.String REPLICATION_FAILED_AT_LIST
public SnapPuller(NamedList initArgs, ReplicationHandler handler, SolrCore sc)
private static org.apache.commons.httpclient.HttpClient createHttpClient(java.lang.String connTimeout, java.lang.String readTimeout)
private void startExecutorService()
NamedList getLatestVersion() throws java.io.IOException
java.io.IOException
NamedList getCommandResponse(NamedList<java.lang.String> commands) throws java.io.IOException
java.io.IOException
private NamedList getNamedListResponse(org.apache.commons.httpclient.methods.PostMethod method) throws java.io.IOException
java.io.IOException
void fetchFileList(long version) throws java.io.IOException
java.io.IOException
boolean fetchLatestIndex(SolrCore core) throws java.io.IOException
java.io.IOException
private void terminateAndWaitFsyncService() throws java.lang.Exception
java.lang.Exception
private void logReplicationTimeAndConfFiles(java.util.Collection<java.util.Map<java.lang.String,java.lang.Object>> modifiedConfFiles, boolean successfulInstall)
static long getTotalBytesDownloaded(SnapPuller snappuller)
private java.lang.StringBuffer readToStringBuffer(long replicationTime, java.lang.String str)
private void doCommit() throws java.io.IOException
java.io.IOException
private java.io.File createTempindexDir(SolrCore core)
private void reloadCore()
private void downloadConfFiles(java.util.List<java.util.Map<java.lang.String,java.lang.Object>> confFilesToDownload, long latestVersion) throws java.lang.Exception
java.lang.Exception
private void downloadIndexFiles(boolean downloadCompleteIndex, java.io.File tmpIdxDir, long latestVersion) throws java.lang.Exception
downloadCompleteIndex
- is it a fresh index copytmpIdxDir
- the directory to which files need to be downloadeed tolatestVersion
- the version numberjava.lang.Exception
private boolean isIndexStale()
private boolean copyAFile(java.io.File tmpIdxDir, java.io.File indexDir, java.lang.String fname, java.util.List<java.lang.String> copiedfiles)
private boolean copyIndexFiles(java.io.File tmpIdxDir, java.io.File indexDir) throws java.io.IOException
java.io.IOException
private void copyTmpConfFiles2Conf(java.io.File tmpconfDir) throws java.io.IOException
java.io.IOException
private java.lang.String getDateAsStr(java.util.Date d)
private boolean modifyIndexProps(java.lang.String tmpIdxDirName)
private java.util.Collection<java.util.Map<java.lang.String,java.lang.Object>> getModifiedConfFiles(java.util.List<java.util.Map<java.lang.String,java.lang.Object>> confFilesToDownload)
confFilesToDownload
- The list of files obtained from masterstatic boolean delTree(java.io.File dir)
void disablePoll()
void enablePoll()
void abortPull()
long getReplicationStartTime()
java.util.List<java.util.Map<java.lang.String,java.lang.Object>> getConfFilesToDownload()
java.util.List<java.util.Map<java.lang.String,java.lang.Object>> getConfFilesDownloaded()
java.util.List<java.util.Map<java.lang.String,java.lang.Object>> getFilesToDownload()
java.util.List<java.util.Map<java.lang.String,java.lang.Object>> getFilesDownloaded()
java.util.Map<java.lang.String,java.lang.Object> getCurrentFile()
boolean isPollingDisabled()
java.lang.Long getNextScheduledExecTime()
private java.io.InputStream checkCompressed(org.apache.commons.httpclient.HttpMethod method, java.io.InputStream respBody) throws java.io.IOException
java.io.IOException
static java.lang.Integer readInterval(java.lang.String interval)
public void destroy()
java.lang.String getMasterUrl()
java.lang.String getPollInterval()