1
0
mirror of https://github.com/winscp/winscp.git synced 2026-10-25 14:19:30 +03:00

Bug 513: Download single large file using multiple SFTP connections

https://winscp.net/tracker/513

Source commit: d57dcad76dfe8b333dfedd8419bce984a181280c
This commit is contained in:
Martin Prikryl
2023-07-26 09:31:50 +00:00
parent cd89d4e431
commit 1aee6f1464
27 changed files with 584 additions and 145 deletions

View File

@@ -321,6 +321,11 @@ UnicodeString DelimitStr(const UnicodeString & Str, wchar_t Quote)
return Result;
}
//---------------------------------------------------------------------------
UnicodeString MidStr(const UnicodeString & Text, int Start)
{
return Text.SubString(Start, Text.Length() - Start + 1);
}
//---------------------------------------------------------------------------
UnicodeString ShellQuoteStr(const UnicodeString & Str)
{
wchar_t Quote = L'"';
@@ -467,8 +472,16 @@ UnicodeString RemoveEmptyLines(const UnicodeString & S)
//---------------------------------------------------------------------------
bool IsNumber(const UnicodeString Str)
{
int Value;
return TryStrToInt(Str, Value);
bool Result = (Str.Length() > 0);
for (int Index = 1; (Index < Str.Length()) && Result; Index++)
{
wchar_t C = Str[Index];
if ((C < L'0') || (C > L'9'))
{
Result = false;
}
}
return Result;
}
//---------------------------------------------------------------------------
UnicodeString Base64ToUrlSafe(const UnicodeString & S)

View File

@@ -58,6 +58,7 @@ UnicodeString CopyToChars(const UnicodeString & Str, int & From, UnicodeString C
UnicodeString CopyToChar(const UnicodeString & Str, wchar_t Ch, bool Trim);
UnicodeString RemoveSuffix(const UnicodeString & Str, const UnicodeString & Suffix, bool RemoveNumbersAfterSuffix = false);
UnicodeString DelimitStr(const UnicodeString & Str, wchar_t Quote = L'"');
UnicodeString MidStr(const UnicodeString & Text, int Start);
UnicodeString ShellQuoteStr(const UnicodeString & Str);
UnicodeString ExceptionLogString(Exception *E);
UnicodeString __fastcall MainInstructions(const UnicodeString & S);

View File

@@ -124,6 +124,8 @@ void __fastcall TConfiguration::Default()
FChecksumCommands = EmptyStr;
FDontReloadMoreThanSessions = 1000;
FScriptProgressFileNameLimit = 25;
FQueueTransfersLimit = 2;
FParallelTransferThreshold = -1; // default (currently off), 0 = explicitly off
FKeyVersion = 0;
CollectUsage = FDefaultCollectUsage;
@@ -260,6 +262,8 @@ UnicodeString __fastcall TConfiguration::PropertyToKey(const UnicodeString & Pro
KEY(String, MimeTypes); \
KEY(Integer, DontReloadMoreThanSessions); \
KEY(Integer, ScriptProgressFileNameLimit); \
KEY(Integer, QueueTransfersLimit); \
KEY(Integer, ParallelTransferThreshold); \
KEY(Integer, KeyVersion); \
KEY(Bool, CollectUsage); \
KEY(String, CertificateStorage); \
@@ -2015,11 +2019,6 @@ UnicodeString __fastcall TConfiguration::GetTimeFormat()
return L"h:nn:ss";
}
//---------------------------------------------------------------------------
UnicodeString __fastcall TConfiguration::GetPartialExt() const
{
return PARTIAL_EXT;
}
//---------------------------------------------------------------------------
UnicodeString __fastcall TConfiguration::GetDefaultKeyFile()
{
return L"";
@@ -2070,6 +2069,11 @@ void __fastcall TConfiguration::SetShowFtpWelcomeMessage(bool value)
SET_CONFIG_PROPERTY(ShowFtpWelcomeMessage);
}
//---------------------------------------------------------------------------
void TConfiguration::SetQueueTransfersLimit(int value)
{
SET_CONFIG_PROPERTY(QueueTransfersLimit);
}
//---------------------------------------------------------------------------
bool __fastcall TConfiguration::GetPersistent()
{
return (Storage != stNul) && !FDontSave;

View File

@@ -80,6 +80,8 @@ private:
int FDontReloadMoreThanSessions;
int FScriptProgressFileNameLimit;
int FKeyVersion;
int FQueueTransfersLimit;
int FParallelTransferThreshold;
UnicodeString FCertificateStorage;
UnicodeString FChecksumCommands;
@@ -131,7 +133,6 @@ private:
UnicodeString __fastcall GetIniFileStorageName(bool ReadingOnly);
void __fastcall SetOptionsStorage(TStrings * value);
TStrings * __fastcall GetOptionsStorage();
UnicodeString __fastcall GetPartialExt() const;
UnicodeString __fastcall GetFileInfoString(const UnicodeString Key);
void __fastcall SetSessionReopenAuto(int value);
void __fastcall SetSessionReopenBackground(int value);
@@ -155,6 +156,7 @@ private:
bool __fastcall GetPersistent();
void SetLocalPortNumberMin(int value);
void SetLocalPortNumberMax(int value);
void SetQueueTransfersLimit(int value);
protected:
TStorage FStorage;
@@ -322,7 +324,6 @@ public:
__property bool ConfirmResume = { read = GetConfirmResume, write = SetConfirmResume};
__property bool AutoReadDirectoryAfterOp = { read = GetAutoReadDirectoryAfterOp, write = SetAutoReadDirectoryAfterOp};
__property bool RememberPassword = { read = GetRememberPassword };
__property UnicodeString PartialExt = {read=GetPartialExt};
__property int SessionReopenAuto = { read = FSessionReopenAuto, write = SetSessionReopenAuto };
__property int SessionReopenBackground = { read = FSessionReopenBackground, write = SetSessionReopenBackground };
__property int SessionReopenTimeout = { read = FSessionReopenTimeout, write = SetSessionReopenTimeout };
@@ -342,6 +343,8 @@ public:
__property UnicodeString MimeTypes = { read = FMimeTypes, write = SetMimeTypes };
__property int DontReloadMoreThanSessions = { read = FDontReloadMoreThanSessions, write = FDontReloadMoreThanSessions };
__property int ScriptProgressFileNameLimit = { read = FScriptProgressFileNameLimit, write = FScriptProgressFileNameLimit };
__property int QueueTransfersLimit = { read = FQueueTransfersLimit, write = SetQueueTransfersLimit };
__property int ParallelTransferThreshold = { read = FParallelTransferThreshold, write = FParallelTransferThreshold };
__property int KeyVersion = { read = FKeyVersion, write = FKeyVersion };
__property UnicodeString TimeFormat = { read = GetTimeFormat };

View File

@@ -58,6 +58,8 @@ void __fastcall TCopyParamType::Default()
ExcludeHiddenFiles = false;
ExcludeEmptyDirectories = false;
Size = -1;
PartOffset = -1;
PartSize = -1;
OnceDoneOperation = odoIdle;
OnTransferOut = NULL;
OnTransferIn = NULL;
@@ -576,6 +578,8 @@ void __fastcall TCopyParamType::Assign(const TCopyParamType * Source)
COPY(ExcludeHiddenFiles);
COPY(ExcludeEmptyDirectories);
COPY(Size);
COPY(PartOffset);
COPY(PartSize);
COPY(OnceDoneOperation);
COPY(OnTransferOut);
COPY(OnTransferIn);
@@ -778,7 +782,7 @@ int __fastcall TCopyParamType::LocalFileAttrs(const TRights & Rights) const
bool __fastcall TCopyParamType::AllowResume(__int64 Size, const UnicodeString & FileName) const
{
bool Result;
if (FileName.Length() + UnicodeString(PARTIAL_EXT).Length() > 255) // it's a different limit than MAX_PATH
if (FileName.Length() + PartialExt.Length() > 255) // it's a different limit than MAX_PATH
{
Result = false;
}
@@ -912,6 +916,8 @@ void __fastcall TCopyParamType::Load(THierarchicalStorage * Storage)
ExcludeHiddenFiles = Storage->ReadBool(L"ExcludeHiddenFiles", ExcludeHiddenFiles);
ExcludeEmptyDirectories = Storage->ReadBool(L"ExcludeEmptyDirectories", ExcludeEmptyDirectories);
Size = -1;
PartOffset = -1;
PartSize = -1;
OnceDoneOperation = odoIdle;
OnTransferOut = NULL;
OnTransferIn = NULL;
@@ -962,6 +968,8 @@ void __fastcall TCopyParamType::Save(THierarchicalStorage * Storage, const TCopy
WRITE_DATA(Bool, ExcludeHiddenFiles);
WRITE_DATA(Bool, ExcludeEmptyDirectories);
DebugAssert(Size < 0);
DebugAssert(PartOffset < 0);
DebugAssert(PartSize < 0);
DebugAssert(OnceDoneOperation == odoIdle);
DebugAssert(OnTransferOut == NULL);
DebugAssert(OnTransferIn == NULL);
@@ -1004,6 +1012,8 @@ bool __fastcall TCopyParamType::operator==(const TCopyParamType & rhp) const
C(ExcludeHiddenFiles) &&
C(ExcludeEmptyDirectories) &&
C(Size) &&
C(PartOffset) &&
C(PartSize) &&
C(OnceDoneOperation) &&
true;
}

View File

@@ -70,6 +70,8 @@ private:
bool FExcludeHiddenFiles;
bool FExcludeEmptyDirectories;
__int64 FSize;
__int64 FPartOffset;
__int64 FPartSize;
TOnceDoneOperation FOnceDoneOperation;
TTransferOutEvent FOnTransferOut;
TTransferInEvent FOnTransferIn;
@@ -145,6 +147,8 @@ public:
__property bool ExcludeHiddenFiles = { read = FExcludeHiddenFiles, write = FExcludeHiddenFiles };
__property bool ExcludeEmptyDirectories = { read = FExcludeEmptyDirectories, write = FExcludeEmptyDirectories };
__property __int64 Size = { read = FSize, write = FSize };
__property __int64 PartSize = { read = FPartSize, write = FPartSize };
__property __int64 PartOffset = { read = FPartOffset, write = FPartOffset };
__property TOnceDoneOperation OnceDoneOperation = { read = FOnceDoneOperation, write = FOnceDoneOperation };
__property TTransferOutEvent OnTransferOut = { read = FOnTransferOut, write = FOnTransferOut };
__property TTransferInEvent OnTransferIn = { read = FOnTransferIn, write = FOnTransferIn };

View File

@@ -1945,6 +1945,7 @@ bool __fastcall TFTPFileSystem::IsCapable(int Capability) const
case fcPreservingTimestampDirs:
case fcResumeSupport:
case fcChangePassword:
case fcParallelFileTransfers:
return false;
default:

View File

@@ -2236,6 +2236,8 @@ TQueueItem * __fastcall TTransferQueueItem::CreateParallelOperation()
DebugAssert(FParallelOperation.get() != NULL);
FParallelOperation->AddClient();
DebugAssert(!FInfo->SingleFile || FParallelOperation->IsParallelFileTransfer);
FInfo->SingleFile = false;
return new TParallelTransferQueueItem(this, FParallelOperation.get());
}
//---------------------------------------------------------------------------

View File

@@ -17,6 +17,8 @@
#include "Cryptography.h"
/* TODO 1 : Path class instead of UnicodeString (handle relativity...) */
//---------------------------------------------------------------------------
const UnicodeString PartialExt(L".filepart");
//---------------------------------------------------------------------------
bool __fastcall IsUnixStyleWindowsPath(const UnicodeString & Path)
{
return (Path.Length() >= 3) && IsLetter(Path[1]) && (Path[2] == L':') && (Path[3] == L'/');
@@ -486,6 +488,28 @@ UnicodeString __fastcall ModificationStr(TDateTime DateTime,
}
}
//---------------------------------------------------------------------------
int GetPartialFileExtLen(const UnicodeString & FileName)
{
int Result = 0;
if (EndsText(PartialExt, FileName))
{
Result = PartialExt.Length();
}
else
{
int P = FileName.LastDelimiter(L".");
if ((P > 0) && (P < FileName.Length()))
{
if (IsNumber(MidStr(FileName, P + 1)) &&
EndsText(PartialExt, FileName.SubString(1, P - 1)))
{
Result = PartialExt.Length() + (FileName.Length() - P + 1);
}
}
}
return Result;
}
//---------------------------------------------------------------------------
int __fastcall FakeFileImageIndex(UnicodeString FileName, unsigned long Attrs,
UnicodeString * TypeName)
{
@@ -501,10 +525,10 @@ int __fastcall FakeFileImageIndex(UnicodeString FileName, unsigned long Attrs,
}
// this should be somewhere else, probably in TUnixDirView,
// as the "partial" overlay is added there too
if (SameText(UnixExtractFileExt(FileName), PARTIAL_EXT))
int PartialFileExtLen = GetPartialFileExtLen(FileName);
if (GetPartialFileExtLen(FileName) > 0)
{
static const size_t PartialExtLen = LENOF(PARTIAL_EXT) - 1;
FileName.SetLength(FileName.Length() - PartialExtLen);
FileName.SetLength(FileName.Length() - PartialFileExtLen);
}
int Icon;

View File

@@ -12,7 +12,7 @@ enum TModificationFmt { mfNone, mfMDHM, mfYMDHM, mfMDY, mfFull };
#define FILETYPE_DEFAULT L'-'
#define FILETYPE_SYMLINK L'L'
#define FILETYPE_DIRECTORY L'D'
#define PARTIAL_EXT L".filepart"
extern const UnicodeString PartialExt;
//---------------------------------------------------------------------------
class TTerminal;
class TRights;
@@ -589,6 +589,7 @@ UnicodeString __fastcall UserModificationStr(TDateTime DateTime,
TModificationFmt Precision);
UnicodeString __fastcall ModificationStr(TDateTime DateTime,
TModificationFmt Precision);
int GetPartialFileExtLen(const UnicodeString & FileName);
int __fastcall FakeFileImageIndex(UnicodeString FileName, unsigned long Attrs = 0,
UnicodeString * TypeName = NULL);
bool __fastcall SameUserName(const UnicodeString & UserName1, const UnicodeString & UserName2);

View File

@@ -820,6 +820,7 @@ bool __fastcall TS3FileSystem::IsCapable(int Capability) const
case fcLocking:
case fcTransferOut:
case fcTransferIn:
case fcParallelFileTransfers:
return false;
default:
@@ -2231,11 +2232,7 @@ void __fastcall TS3FileSystem::Sink(
if (DeleteLocalFile)
{
FILE_OPERATION_LOOP_BEGIN
{
THROWOSIFFALSE(Sysutils::DeleteFile(ApiPath(DestFullName)));
}
FILE_OPERATION_LOOP_END(FMTLOAD(DELETE_LOCAL_FILE_ERROR, (DestFullName)));
FTerminal->DoDeleteLocalFile(DestFullName);
}
}
}

View File

@@ -461,6 +461,7 @@ bool __fastcall TSCPFileSystem::IsCapable(int Capability) const
case fcResumeSupport:
case fcSkipTransfer:
case fcParallelTransfers: // does not implement cpNoRecurse
case fcParallelFileTransfers:
case fcTransferOut:
case fcTransferIn:
return false;

View File

@@ -45,7 +45,9 @@ enum TFSCapability { fcUserGroupListing, fcModeChanging, fcAclChangingFiles, fcG
fcModeChangingUpload, fcPreservingTimestampUpload, fcShellAnyCommand,
fcSecondaryShell, fcRemoveCtrlZUpload, fcRemoveBOMUpload, fcMoveToQueue,
fcLocking, fcPreservingTimestampDirs, fcResumeSupport,
fcChangePassword, fcSkipTransfer, fcParallelTransfers, fcBackgroundTransfers,
fcChangePassword, fcSkipTransfer,
fcParallelTransfers, fcParallelFileTransfers,
fcBackgroundTransfers,
fcTransferOut, fcTransferIn,
fcMoveOverExistingFile,
fcCount };

View File

@@ -1437,11 +1437,12 @@ public:
}
virtual __fastcall ~TSFTPDownloadQueue(){}
bool __fastcall Init(int QueueLen, const RawByteString & AHandle,__int64 ATransferred,
TFileOperationProgressType * AOperationProgress)
bool __fastcall Init(
int QueueLen, const RawByteString & AHandle, __int64 ATransferred, __int64 PartSize, TFileOperationProgressType * AOperationProgress)
{
FHandle = AHandle;
FTransferred = ATransferred;
FPartSize = PartSize;
OperationProgress = AOperationProgress;
return TSFTPFixedLenQueue::Init(QueueLen);
@@ -1465,10 +1466,23 @@ protected:
virtual bool __fastcall InitRequest(TSFTPQueuePacket * Request)
{
unsigned int BlockSize = FFileSystem->DownloadBlockSize(OperationProgress);
InitRequest(Request, FTransferred, BlockSize);
Request->Token = reinterpret_cast<void*>(BlockSize);
FTransferred += BlockSize;
return true;
if (FPartSize >= 0)
{
__int64 Remaining = FPartSize - FTransferred;
if (Remaining < BlockSize)
{
// It's lower, so the cast is safe
BlockSize = static_cast<unsigned int>(Remaining);
}
}
bool Result = (BlockSize > 0);
if (Result)
{
InitRequest(Request, FTransferred, BlockSize);
Request->Token = reinterpret_cast<void*>(BlockSize);
FTransferred += BlockSize;
}
return Result;
}
void __fastcall InitRequest(TSFTPPacket * Request, __int64 Offset,
@@ -1488,6 +1502,7 @@ protected:
private:
TFileOperationProgressType * OperationProgress;
__int64 FTransferred;
__int64 FPartSize;
RawByteString FHandle;
};
//---------------------------------------------------------------------------
@@ -1548,20 +1563,20 @@ protected:
if (Result)
{
bool Last;
if (FOnTransferIn != NULL)
{
size_t Read = BlockBuf.LoadFromIn(FOnTransferIn, FTerminal, BlockSize);
Last = (Read < BlockSize);
}
else
if (FOnTransferIn != NULL)
{
size_t Read = BlockBuf.LoadFromIn(FOnTransferIn, FTerminal, BlockSize);
Last = (Read < BlockSize);
}
else
{
FILE_OPERATION_LOOP_BEGIN
{
BlockBuf.LoadStream(FStream, BlockSize, false);
}
FILE_OPERATION_LOOP_END(FMTLOAD(READ_ERROR, (FFileName)));
Last = (FStream->Position >= FStream->Size);
}
Last = (FStream->Position >= FStream->Size);
}
FEnd = (BlockBuf.Size == 0);
Result = !FEnd;
@@ -2017,7 +2032,7 @@ const TFileSystemInfo & __fastcall TSFTPFileSystem::GetFileSystemInfo(bool /*Ret
//---------------------------------------------------------------------------
bool __fastcall TSFTPFileSystem::TemporaryTransferFile(const UnicodeString & FileName)
{
return SameText(UnixExtractFileExt(FileName), PARTIAL_EXT);
return (GetPartialFileExtLen(FileName) > 0);
}
//---------------------------------------------------------------------------
bool __fastcall TSFTPFileSystem::GetStoredCredentialsTried()
@@ -2095,6 +2110,7 @@ bool __fastcall TSFTPFileSystem::IsCapable(int Capability) const
case fcResumeSupport:
case fcSkipTransfer:
case fcParallelTransfers:
case fcParallelFileTransfers:
return !FTerminal->IsEncryptingFiles();
case fcRename:
@@ -4587,7 +4603,7 @@ void __fastcall TSFTPFileSystem::Source(
if (ResumeAllowed)
{
DestPartialFullName = DestFullName + FTerminal->Configuration->PartialExt;
DestPartialFullName = DestFullName + PartialExt;
if (FLAGCLEAR(Flags, tfNewDirectory))
{
@@ -4680,7 +4696,7 @@ void __fastcall TSFTPFileSystem::Source(
{
// update paths in case user changes the file name
DestFullName = LocalCanonify(TargetDir + DestFileName);
DestPartialFullName = DestFullName + FTerminal->Configuration->PartialExt;
DestPartialFullName = DestFullName + PartialExt;
FTerminal->LogEvent(L"Checking existence of new file.");
DestFileExists = RemoteFileExists(DestFullName, NULL);
}
@@ -5346,7 +5362,8 @@ void __fastcall TSFTPFileSystem::Sink(
!OperationProgress->AsciiTransfer &&
CopyParam->AllowResume(OperationProgress->TransferSize, DestFileName) &&
!FTerminal->IsEncryptingFiles() &&
(CopyParam->OnTransferOut == NULL);
(CopyParam->OnTransferOut == NULL) &&
(CopyParam->PartOffset < 0);
HANDLE LocalHandle = NULL;
TStream * FileStream = NULL;
@@ -5363,7 +5380,7 @@ void __fastcall TSFTPFileSystem::Sink(
if (ResumeAllowed)
{
DestPartialFullName = DestFullName + FTerminal->Configuration->PartialExt;
DestPartialFullName = DestFullName + PartialExt;
LocalFileName = DestPartialFullName;
FTerminal->LogEvent(L"Checking existence of partially transferred file.");
@@ -5392,11 +5409,7 @@ void __fastcall TSFTPFileSystem::Sink(
{
CloseHandle(LocalHandle);
LocalHandle = NULL;
FILE_OPERATION_LOOP_BEGIN
{
THROWOSIFFALSE(Sysutils::DeleteFile(ApiPath(DestPartialFullName)));
}
FILE_OPERATION_LOOP_END(FMTLOAD(DELETE_LOCAL_FILE_ERROR, (DestPartialFullName)));
FTerminal->DoDeleteLocalFile(DestPartialFullName);
}
else
{
@@ -5462,16 +5475,12 @@ void __fastcall TSFTPFileSystem::Sink(
if (PrevDestFileName != DestFileName)
{
DestFullName = TargetDir + DestFileName;
DestPartialFullName = DestFullName + FTerminal->Configuration->PartialExt;
DestPartialFullName = DestFullName + PartialExt;
if (ResumeAllowed)
{
if (FileExists(ApiPath(DestPartialFullName)))
{
FILE_OPERATION_LOOP_BEGIN
{
THROWOSIFFALSE(Sysutils::DeleteFile(ApiPath(DestPartialFullName)));
}
FILE_OPERATION_LOOP_END(FMTLOAD(DELETE_LOCAL_FILE_ERROR, (DestPartialFullName)));
FTerminal->DoDeleteLocalFile(DestPartialFullName);
}
LocalFileName = DestPartialFullName;
}
@@ -5550,7 +5559,8 @@ void __fastcall TSFTPFileSystem::Sink(
{
QueueLen = 1;
}
Queue.Init(QueueLen, RemoteHandle, OperationProgress->TransferredSize, OperationProgress);
__int64 Offset = OperationProgress->TransferredSize + std::max(CopyParam->PartOffset, 0LL);
Queue.Init(QueueLen, RemoteHandle, Offset, CopyParam->PartSize, OperationProgress);
bool Eof = false;
bool PrevIncomplete = false;
@@ -5567,7 +5577,7 @@ void __fastcall TSFTPFileSystem::Sink(
{
if (Missing > 0)
{
Queue.InitFillGapRequest(OperationProgress->TransferredSize, Missing, &DataPacket);
Queue.InitFillGapRequest(Offset + OperationProgress->TransferredSize, Missing, &DataPacket);
GapFillCount++;
SendPacketAndReceiveResponse(&DataPacket, &DataPacket, SSH_FXP_DATA, asEOF);
}
@@ -5641,6 +5651,12 @@ void __fastcall TSFTPFileSystem::Sink(
Eof = DataPacket.GetBool();
}
if ((CopyParam->PartSize >= 0) &&
(OperationProgress->TransferredSize >= CopyParam->PartSize))
{
Eof = true;
}
if (OperationProgress->AsciiTransfer)
{
DebugAssert(!ResumeTransfer && !ResumeAllowed);
@@ -5705,6 +5721,7 @@ void __fastcall TSFTPFileSystem::Sink(
if (ResumeAllowed)
{
// See also DoRenameLocalFileForce
FILE_OPERATION_LOOP_BEGIN
{
if (FileExists(ApiPath(DestFullName)))
@@ -5737,14 +5754,11 @@ void __fastcall TSFTPFileSystem::Sink(
if (DeleteLocalFile && (!ResumeAllowed || OperationProgress->LocallyUsed == 0) &&
(OverwriteMode == omOverwrite))
{
FILE_OPERATION_LOOP_BEGIN
{
THROWOSIFFALSE(Sysutils::DeleteFile(ApiPath(LocalFileName)));
}
FILE_OPERATION_LOOP_END(FMTLOAD(DELETE_LOCAL_FILE_ERROR, (LocalFileName)));
FTerminal->DoDeleteLocalFile(LocalFileName);
}
// if the transfer was finished, the file is closed already
// if the transfer was finished, the file is usually closed already
// (except for special cases like SFTPv6 EOF indication or partial file downlaod)
if (FTerminal->Active && !RemoteHandle.IsEmpty())
{
// do not wait for response

View File

@@ -699,7 +699,8 @@ TParallelOperation::TParallelOperation(TOperationSide Side)
//---------------------------------------------------------------------------
void TParallelOperation::Init(
TStrings * AFileList, const UnicodeString & TargetDir, const TCopyParamType * CopyParam, int Params,
TFileOperationProgressType * MainOperationProgress, const UnicodeString & MainName)
TFileOperationProgressType * MainOperationProgress, const UnicodeString & MainName,
__int64 ParallelFileSize)
{
DebugAssert(FFileList.get() == NULL);
// More lists should really happen in scripting only, which does not support parallel transfers atm.
@@ -714,6 +715,12 @@ void TParallelOperation::Init(
FMainName = MainName;
FListIndex = 0;
FIndex = 0;
FIsParallelFileTransfer = (ParallelFileSize >= 0);
FParallelFileSize = ParallelFileSize;
FParallelFileOffset = 0;
FParallelFileCount = 0;
FParallelFileMerging = false;
FParallelFileMerged = 0;
}
//---------------------------------------------------------------------------
TParallelOperation::~TParallelOperation()
@@ -786,7 +793,9 @@ void TParallelOperation::WaitFor()
}
//---------------------------------------------------------------------------
void TParallelOperation::Done(const UnicodeString & FileName, bool Dir, bool Success)
void TParallelOperation::Done(
const UnicodeString & FileName, bool Dir, bool Success, const UnicodeString & TargetDir,
const TCopyParamType * CopyParam, TTerminal * Terminal)
{
if (Dir)
{
@@ -844,6 +853,113 @@ void TParallelOperation::Done(const UnicodeString & FileName, bool Dir, bool Suc
}
}
}
else
{
if (IsParallelFileTransfer)
{
TGuard Guard(FSection.get());
try
{
if (DebugAlwaysTrue(FSide == osRemote))
{
TParallelFileOffsets::const_iterator I = std::find(FParallelFileOffsets.begin(), FParallelFileOffsets.end(), CopyParam->PartOffset);
if (DebugAlwaysTrue(I != FParallelFileOffsets.end()))
{
int Index = I - FParallelFileOffsets.begin();
DebugAssert(!FParallelFileDones[Index]);
FParallelFileDones[Index] = true;
if (!FParallelFileMerging)
{
TFileOperationProgressType * OperationProgress = Terminal->OperationProgress;
// Once we obtain "merging" semaphor, we won't leave until everything is merged
TAutoFlag MergingFlag(FParallelFileMerging);
try
{
UnicodeString TargetName = TPath::Combine(TargetDir, FParallelFileTargetName);
UnicodeString TargetNamePartial = TargetName + PartialExt;
UnicodeString TargetNamePartialOnly = ExtractFileName(TargetNamePartial);
while (true)
{
if ((FParallelFileMerged >= FParallelFileCount) || !FParallelFileDones[FParallelFileMerged])
{
break;
}
else
{
TUnguard Unguard(FSection.get());
UnicodeString FileNameOnly = UnixExtractFileName(FileName);
// Safe as write access to FParallelFileMerged is guarded by FParallelFileMerging
int Index = FParallelFileMerged;
UnicodeString TargetPartName = GetPartPrefix(TPath::Combine(TargetDir, FileNameOnly)) + IntToStr(Index);
if ((CopyParam->PartSize >= 0) && (Terminal->OperationProgress->TransferredSize != CopyParam->PartSize))
{
UnicodeString Message =
FMTLOAD(INCONSISTENT_SIZE, (TargetPartName, Terminal->OperationProgress->TransferredSize, CopyParam->PartSize));
Terminal->TerminalError(NULL, Message);
}
if (Index == 0)
{
Terminal->LogEvent(FORMAT(L"Renaming part %d of \"%s\" to \"%s\"...", (Index, FileNameOnly, TargetNamePartialOnly)));
Terminal->DoRenameLocalFileForce(TargetPartName, TargetNamePartial);
}
else
{
{
Terminal->LogEvent(FORMAT(L"Appending part %d of \"%s\" to \"%s\"...", (Index, FileNameOnly, TargetNamePartialOnly)));
std::unique_ptr<TFileStream> SourceStream(new TFileStream(ApiPath(TargetPartName), fmOpenRead | fmShareDenyWrite));
std::unique_ptr<TFileStream> DestStream(new TFileStream(ApiPath(TargetNamePartial), fmOpenWrite | fmShareDenyWrite));
HANDLE DestHandle = reinterpret_cast<HANDLE>(DestStream->Handle);
FILETIME WrTime;
bool GotWrTime = GetFileTime(DestHandle, NULL, NULL, &WrTime);
DestStream->Seek(0L, soEnd);
DestStream->CopyFrom(SourceStream.get(), SourceStream->Size);
if (GotWrTime)
{
SetFileTime(DestHandle, NULL, NULL, &WrTime);
}
}
Terminal->DoDeleteLocalFile(TargetPartName);
}
FParallelFileMerged++;
}
}
if ((FParallelFileMerged == FParallelFileCount) && (FParallelFileOffset == FParallelFileSize))
{
Terminal->LogEvent(FORMAT(L"Renaming completed \"%s\" to \"%s\"...", (TargetNamePartialOnly, FParallelFileTargetName)));
Terminal->DoRenameLocalFileForce(TargetNamePartial, TargetName);
}
}
catch (...)
{
Success = false;
throw;
}
}
}
}
}
__finally
{
if (!Success)
{
// If any part fails, cancel whole transfer.
// We should get here only after the user sees/acknowledges the failure.
FMainOperationProgress->SetCancel(csCancel);
}
}
}
}
}
//---------------------------------------------------------------------------
bool TParallelOperation::CheckEnd(TCollectedFileList * Files)
@@ -857,12 +973,40 @@ bool TParallelOperation::CheckEnd(TCollectedFileList * Files)
return Result;
}
//---------------------------------------------------------------------------
TCollectedFileList * TParallelOperation::GetFileList(int Index)
bool TParallelOperation::GetOnlyFile(TStrings * FileList, UnicodeString & FileName, TObject *& Object)
{
return DebugNotNull(dynamic_cast<TCollectedFileList *>(FFileList->Objects[Index]));
bool Result = (FileList->Count == 1);
if (Result)
{
TCollectedFileList * OnlyFileList = GetFileList(FileList, 0);
Result = (OnlyFileList->Count() == 1) && !OnlyFileList->IsDir(0);
if (Result)
{
FileName = OnlyFileList->GetFileName(0);
Object = OnlyFileList->GetObject(0);
}
}
return Result;
}
//---------------------------------------------------------------------------
int TParallelOperation::GetNext(TTerminal * Terminal, UnicodeString & FileName, TObject *& Object, UnicodeString & TargetDir, bool & Dir, bool & Recursed)
TCollectedFileList * TParallelOperation::GetFileList(TStrings * FileList, int Index)
{
return DebugNotNull(dynamic_cast<TCollectedFileList *>(FileList->Objects[Index]));
}
//---------------------------------------------------------------------------
TCollectedFileList * TParallelOperation::GetFileList(int Index)
{
return GetFileList(FFileList.get(), Index);
}
//---------------------------------------------------------------------------
UnicodeString TParallelOperation::GetPartPrefix(const UnicodeString & FileName)
{
return FORMAT(L"%s%s.", (FileName, PartialExt));
}
//---------------------------------------------------------------------------
int TParallelOperation::GetNext(
TTerminal * Terminal, UnicodeString & FileName, TObject *& Object, UnicodeString & TargetDir, bool & Dir,
bool & Recursed, TCopyParamType *& CustomCopyParam)
{
TGuard Guard(FSection.get());
int Result = 1;
@@ -931,11 +1075,9 @@ int TParallelOperation::GetNext(TTerminal * Terminal, UnicodeString & FileName,
if (!TargetDir.IsEmpty())
{
if (Dir)
UnicodeString OnlyFileName;
if (Dir || IsParallelFileTransfer) // optimization
{
TDirectoryData DirectoryData;
UnicodeString OnlyFileName;
if (FSide == osLocal)
{
OnlyFileName = ExtractFileName(FileName);
@@ -945,6 +1087,12 @@ int TParallelOperation::GetNext(TTerminal * Terminal, UnicodeString & FileName,
OnlyFileName = UnixExtractFileName(FileName);
}
OnlyFileName = Terminal->ChangeFileName(FCopyParam, OnlyFileName, FSide, FirstLevel);
}
if (Dir)
{
DebugAssert(!OnlyFileName.IsEmpty());
TDirectoryData DirectoryData;
if (FSide == osLocal)
{
DirectoryData.OppositePath = UnixCombinePaths(TargetDir, OnlyFileName);
@@ -959,11 +1107,49 @@ int TParallelOperation::GetNext(TTerminal * Terminal, UnicodeString & FileName,
FDirectories.insert(std::make_pair(FileName, DirectoryData));
}
// The current implementation of UpdateFileList relies on this specific way of changing state
// (all files before FIndex one-by-one)
Files->SetState(FIndex, qfsProcessed);
FIndex++;
CheckEnd(Files);
bool Processed = true;
if (IsParallelFileTransfer)
{
CustomCopyParam = new TCopyParamType(*FCopyParam);
CustomCopyParam->PartOffset = FParallelFileOffset;
__int64 Remaining = FParallelFileSize - CustomCopyParam->PartOffset;
CustomCopyParam->PartSize = FParallelFileSize / Configuration->QueueTransfersLimit;
DebugAssert(!OnlyFileName.IsEmpty());
if (FParallelFileTargetName.IsEmpty())
{
FParallelFileTargetName = OnlyFileName;
}
DebugAssert(FParallelFileTargetName == OnlyFileName);
int Index = FParallelFileCount;
UnicodeString PartFileName = GetPartPrefix(OnlyFileName) + IntToStr(Index);
FParallelFileCount++;
FParallelFileOffsets.push_back(CustomCopyParam->PartOffset);
FParallelFileDones.push_back(false);
DebugAssert(FParallelFileOffsets.size() == static_cast<size_t>(FParallelFileCount));
CustomCopyParam->FileMask = DelimitFileNameMask(PartFileName);
if ((CustomCopyParam->PartSize >= Remaining) ||
(Remaining - CustomCopyParam->PartSize < CustomCopyParam->PartSize / 10))
{
CustomCopyParam->PartSize = -1; // Until the end
FParallelFileOffset = FParallelFileSize;
Terminal->LogEvent(FORMAT(L"Starting transfer of \"%s\" part %d from %s until the EOF", (OnlyFileName, Index, IntToStr(CustomCopyParam->PartOffset))));
}
else
{
Processed = false;
FParallelFileOffset += CustomCopyParam->PartSize;
Terminal->LogEvent(FORMAT(L"Starting transfer of \"%s\" part %d from %s, length %s", (OnlyFileName, Index, IntToStr(CustomCopyParam->PartOffset), IntToStr(CustomCopyParam->PartSize))));
}
}
if (Processed)
{
// The current implementation of UpdateFileList relies on this specific way of changing state
// (all files before FIndex one-by-one)
Files->SetState(FIndex, qfsProcessed);
FIndex++;
CheckEnd(Files);
}
}
}
@@ -2866,7 +3052,8 @@ TBatchOverwrite __fastcall TTerminal::EffectiveBatchOverwrite(
{
TBatchOverwrite Result;
if (Special &&
(FLAGSET(Params, cpResume) || CopyParam->ResumeTransfer(SourceFullFileName)))
(FLAGSET(Params, cpResume) || CopyParam->ResumeTransfer(SourceFullFileName)) &&
(CopyParam->PartOffset < 0))
{
Result = boResume;
}
@@ -2876,7 +3063,8 @@ TBatchOverwrite __fastcall TTerminal::EffectiveBatchOverwrite(
}
else if (CopyParam->NewerOnly &&
(((OperationProgress->Side == osLocal) && IsCapable[fcNewerOnlyUpload]) ||
(OperationProgress->Side != osLocal)))
(OperationProgress->Side != osLocal)) &&
DebugAlwaysTrue(CopyParam->PartOffset < 0))
{
// no way to change batch overwrite mode when CopyParam->NewerOnly is on
Result = boOlder;
@@ -6894,7 +7082,10 @@ int __fastcall TTerminal::CopyToParallel(TParallelOperation * ParallelOperation,
bool Dir;
bool Recursed;
int Result = ParallelOperation->GetNext(this, FileName, Object, TargetDir, Dir, Recursed);
TCopyParamType * CustomCopyParam = NULL;
int Result = ParallelOperation->GetNext(this, FileName, Object, TargetDir, Dir, Recursed, CustomCopyParam);
std::unique_ptr<TCopyParamType> CustomCopyParamOwner(CustomCopyParam);
if (Result > 0)
{
std::unique_ptr<TStrings> FilesToCopy(new TStringList());
@@ -6913,24 +7104,25 @@ int __fastcall TTerminal::CopyToParallel(TParallelOperation * ParallelOperation,
int Prev = OperationProgress->FilesFinishedSuccessfully;
DebugAssert((FOperationProgress == OperationProgress) || (FOperationProgress == NULL));
TFileOperationProgressType * PrevOperationProgress = FOperationProgress;
const TCopyParamType * CopyParam = (CustomCopyParam != NULL) ? CustomCopyParam : ParallelOperation->CopyParam;
try
{
FOperationProgress = OperationProgress;
if (ParallelOperation->Side == osLocal)
{
FFileSystem->CopyToRemote(
FilesToCopy.get(), TargetDir, ParallelOperation->CopyParam, Params, OperationProgress, OnceDoneOperation);
FilesToCopy.get(), TargetDir, CopyParam, Params, OperationProgress, OnceDoneOperation);
}
else if (DebugAlwaysTrue(ParallelOperation->Side == osRemote))
{
FFileSystem->CopyToLocal(
FilesToCopy.get(), TargetDir, ParallelOperation->CopyParam, Params, OperationProgress, OnceDoneOperation);
FilesToCopy.get(), TargetDir, CopyParam, Params, OperationProgress, OnceDoneOperation);
}
}
__finally
{
bool Success = (Prev < OperationProgress->FilesFinishedSuccessfully);
ParallelOperation->Done(FileName, Dir, Success);
ParallelOperation->Done(FileName, Dir, Success, TargetDir, CopyParam, this);
// Not to fail an assertion in OperationStop when called from CopyToRemote or CopyToLocal,
// when FOperationProgress is already OperationProgress.
FOperationProgress = PrevOperationProgress;
@@ -7093,7 +7285,7 @@ bool __fastcall TTerminal::CopyToRemote(
if (Parallel)
{
// OnceDoneOperation is not supported
ParallelOperation->Init(Files.release(), TargetDir, CopyParam, Params, &OperationProgress, Log->Name);
ParallelOperation->Init(Files.release(), TargetDir, CopyParam, Params, &OperationProgress, Log->Name, -1);
CopyParallel(ParallelOperation, &OperationProgress);
}
else
@@ -7371,11 +7563,23 @@ void __fastcall TTerminal::DirectorySource(
}
}
//---------------------------------------------------------------------------
bool TTerminal::UseAsciiTransfer(
const UnicodeString & BaseFileName, TOperationSide Side, const TCopyParamType * CopyParam,
const TFileMasks::TParams & MaskParams)
{
return
IsCapable[fcTextMode] &&
CopyParam->UseAsciiTransfer(BaseFileName, Side, MaskParams) &&
// Used either before PartSize is set (to check that file is to be transfered in binary mode, hence allows parallel file transfer),
// or later during parallel transfer (which never happens for ascii mode).
DebugAlwaysTrue(CopyParam->PartSize < 0);
}
//---------------------------------------------------------------------------
void __fastcall TTerminal::SelectTransferMode(
const UnicodeString & BaseFileName, TOperationSide Side, const TCopyParamType * CopyParam,
const TFileMasks::TParams & MaskParams)
{
bool AsciiTransfer = IsCapable[fcTextMode] && CopyParam->UseAsciiTransfer(BaseFileName, Side, MaskParams);
bool AsciiTransfer = UseAsciiTransfer(BaseFileName, Side, CopyParam, MaskParams);
OperationProgress->SetAsciiTransfer(AsciiTransfer);
UnicodeString ModeName = (OperationProgress->AsciiTransfer ? L"Ascii" : L"Binary");
LogEvent(0, FORMAT(L"%s transfer mode selected.", (ModeName)));
@@ -7391,6 +7595,29 @@ void __fastcall TTerminal::SelectSourceTransferMode(const TLocalFileHandle & Han
SelectTransferMode(BaseFileName, osLocal, CopyParam, MaskParams);
}
//---------------------------------------------------------------------------
void TTerminal::DoDeleteLocalFile(const UnicodeString & FileName)
{
FILE_OPERATION_LOOP_BEGIN
{
DeleteFileChecked(FileName);
}
FILE_OPERATION_LOOP_END(FMTLOAD(DELETE_LOCAL_FILE_ERROR, (FileName)));
}
//---------------------------------------------------------------------------
void TTerminal::DoRenameLocalFileForce(const UnicodeString & OldName, const UnicodeString & NewName)
{
if (::FileExists(ApiPath(NewName)))
{
DoDeleteLocalFile(NewName);
}
FILE_OPERATION_LOOP_BEGIN
{
THROWOSIFFALSE(Sysutils::RenameFile(ApiPath(OldName), ApiPath(NewName)));
}
FILE_OPERATION_LOOP_END(FMTLOAD(RENAME_FILE_ERROR, (OldName, NewName)));
}
//---------------------------------------------------------------------------
void __fastcall TTerminal::UpdateSource(const TLocalFileHandle & Handle, const TCopyParamType * CopyParam, int Params)
{
// TODO: Delete also read-only files.
@@ -7400,11 +7627,7 @@ void __fastcall TTerminal::UpdateSource(const TLocalFileHandle & Handle, const T
{
LogEvent(FORMAT(L"Deleting successfully uploaded source file \"%s\".", (Handle.FileName)));
FILE_OPERATION_LOOP_BEGIN
{
THROWOSIFFALSE(Sysutils::DeleteFile(ApiPath(Handle.FileName)));
}
FILE_OPERATION_LOOP_END(FMTLOAD(DELETE_LOCAL_FILE_ERROR, (Handle.FileName)));
DoDeleteLocalFile(Handle.FileName);
}
}
else if (CopyParam->ClearArchive && FLAGSET(Handle.Attrs, faArchive))
@@ -7492,6 +7715,62 @@ void __fastcall TTerminal::Source(
UpdateSource(Handle, CopyParam, Params);
}
//---------------------------------------------------------------------------
void TTerminal::CheckParallelFileTransfer(
const UnicodeString & TargetDir, TStringList * Files, const TCopyParamType * CopyParam, int Params,
UnicodeString & ParallelFileName, __int64 & ParallelFileSize, TFileOperationProgressType * OperationProgress)
{
if ((Configuration->ParallelTransferThreshold > 0) &&
FFileSystem->IsCapable(fcParallelFileTransfers))
{
TObject * ParallelObject = NULL;
if (TParallelOperation::GetOnlyFile(Files, ParallelFileName, ParallelObject))
{
TRemoteFile * File = static_cast<TRemoteFile *>(ParallelObject);
const TRemoteFile * UltimateFile = File->Resolve();
if ((UltimateFile == File) && // not tested with symlinks
(UltimateFile->Size >= static_cast<__int64>(Configuration->ParallelTransferThreshold) * 1024))
{
UnicodeString BaseFileName = GetBaseFileName(ParallelFileName);
TFileMasks::TParams MaskParams;
MaskParams.Size = UltimateFile->Size;
MaskParams.Modification = File->Modification;
if (!UseAsciiTransfer(BaseFileName, osRemote, CopyParam, MaskParams))
{
ParallelFileSize = UltimateFile->Size;
UnicodeString TargetFileName = CopyParam->ChangeFileName(UnixExtractFileName(ParallelFileName), osRemote, true);
UnicodeString DestFullName = TPath::Combine(TargetDir, TargetFileName);
if (::FileExists(ApiPath(DestFullName)))
{
TSuspendFileOperationProgress Suspend(OperationProgress);
TOverwriteFileParams FileParams;
__int64 MTime;
OpenLocalFile(DestFullName, GENERIC_READ, NULL, NULL, NULL, &MTime, NULL, &FileParams.DestSize, false);
FileParams.SourceSize = ParallelFileSize;
FileParams.SourceTimestamp = UltimateFile->Modification;
FileParams.SourcePrecision = UltimateFile->ModificationFmt;
FileParams.DestTimestamp = UnixToDateTime(MTime, SessionData->DSTMode);
int Answers = qaYes | qaNo | qaCancel;
TQueryParams QueryParams(qpNeverAskAgainCheck);
unsigned int Answer =
ConfirmFileOverwrite(
ParallelFileName, TargetFileName, &FileParams, Answers, &QueryParams, osRemote,
CopyParam, Params, OperationProgress, EmptyStr);
switch (Answer)
{
case qaCancel:
case qaNo:
OperationProgress->SetCancelAtLeast(csCancel);
break;
}
}
}
}
}
}
}
//---------------------------------------------------------------------------
bool __fastcall TTerminal::CopyToLocal(
TStrings * FilesToCopy, const UnicodeString & TargetDir, const TCopyParamType * CopyParam, int Params,
TParallelOperation * ParallelOperation)
@@ -7521,7 +7800,9 @@ bool __fastcall TTerminal::CopyToLocal(
TFileOperationProgressType OperationProgress(&DoProgress, &DoFinished);
std::unique_ptr<TStringList> Files;
bool ACanParallel = CanParallel(CopyParam, Params, ParallelOperation);
bool ACanParallel =
CanParallel(CopyParam, Params, ParallelOperation) &&
DebugAlwaysTrue(CopyParam->OnTransferOut == NULL);
if (ACanParallel)
{
Files.reset(new TStringList());
@@ -7586,9 +7867,24 @@ bool __fastcall TTerminal::CopyToLocal(
if (Parallel)
{
// OnceDoneOperation is not supported
ParallelOperation->Init(Files.release(), TargetDir, CopyParam, Params, &OperationProgress, Log->Name);
CopyParallel(ParallelOperation, &OperationProgress);
UnicodeString ParallelFileName;
__int64 ParallelFileSize = -1;
CheckParallelFileTransfer(TargetDir, Files.get(), CopyParam, Params, ParallelFileName, ParallelFileSize, &OperationProgress);
if (OperationProgress.Cancel == csContinue)
{
if (ParallelFileSize >= 0)
{
DebugAssert(ParallelFileSize == TotalSize);
Params |= cpNoConfirmation;
}
// OnceDoneOperation is not supported
ParallelOperation->Init(
Files.release(), TargetDir, CopyParam, Params, &OperationProgress, Log->Name, ParallelFileSize);
UnicodeString ParallelFilePrefix;
CopyParallel(ParallelOperation, &OperationProgress);
}
}
else
{
@@ -7867,7 +8163,7 @@ void __fastcall TTerminal::Sink(
// Suppose same data size to transfer as to write
// (not true with ASCII transfer)
__int64 TransferSize = UltimateFile->Size;
__int64 TransferSize = (CopyParam->PartSize >= 0) ? CopyParam->PartSize : (UltimateFile->Size - std::max(0LL, CopyParam->PartOffset));
OperationProgress->SetLocalSize(TransferSize);
if (IsFileEncrypted(FileName))
{

View File

@@ -156,6 +156,7 @@ friend class TTunnelUI;
friend class TCallbackGuard;
friend class TSecondaryTerminal;
friend class TRetryOperationLoop;
friend class TParallelOperation;
private:
TSessionData * FSessionData;
@@ -473,10 +474,15 @@ protected:
const UnicodeString & DirectoryName, const UnicodeString & TargetDir, const UnicodeString & DestDirectoryName,
int Attrs, const TCopyParamType * CopyParam, int Params,
TFileOperationProgressType * OperationProgress, unsigned int Flags);
bool UseAsciiTransfer(
const UnicodeString & BaseFileName, TOperationSide Side, const TCopyParamType * CopyParam,
const TFileMasks::TParams & MaskParams);
void __fastcall SelectTransferMode(
const UnicodeString & BaseFileName, TOperationSide Side, const TCopyParamType * CopyParam,
const TFileMasks::TParams & MaskParams);
void __fastcall SelectSourceTransferMode(const TLocalFileHandle & Handle, const TCopyParamType * CopyParam);
void DoDeleteLocalFile(const UnicodeString & FileName);
void DoRenameLocalFileForce(const UnicodeString & OldName, const UnicodeString & NewName);
void __fastcall UpdateSource(const TLocalFileHandle & Handle, const TCopyParamType * CopyParam, int Params);
void __fastcall DoCopyToLocal(
TStrings * FilesToCopy, const UnicodeString & TargetDir, const TCopyParamType * CopyParam, int Params,
@@ -492,6 +498,9 @@ protected:
void __fastcall UpdateTargetAttrs(
const UnicodeString & DestFullName, const TRemoteFile * File, const TCopyParamType * CopyParam, int Attrs);
void __fastcall UpdateTargetTime(HANDLE Handle, TDateTime Modification, TDSTMode DSTMode);
void CheckParallelFileTransfer(
const UnicodeString & TargetDir, TStringList * Files, const TCopyParamType * CopyParam, int Params,
UnicodeString & ParallelFileName, __int64 & ParallelFileSize, TFileOperationProgressType * OperationProgress);
TRemoteFile * CheckRights(const UnicodeString & EntryType, const UnicodeString & FileName, bool & WrongRights);
bool IsValidFile(TRemoteFile * File);
void __fastcall CalculateSubFoldersChecksum(
@@ -866,7 +875,8 @@ public:
void Init(
TStrings * AFiles, const UnicodeString & TargetDir, const TCopyParamType * CopyParam, int Params,
TFileOperationProgressType * MainOperationProgress, const UnicodeString & MainName);
TFileOperationProgressType * MainOperationProgress, const UnicodeString & MainName,
__int64 ParallelFileSize);
bool IsInitialized();
void WaitFor();
@@ -874,17 +884,23 @@ public:
void AddClient();
void RemoveClient();
int GetNext(
TTerminal * Terminal, UnicodeString & FileName, TObject *& Object, UnicodeString & TargetDir,
bool & Dir, bool & Recursed);
void Done(const UnicodeString & FileName, bool Dir, bool Success);
TTerminal * Terminal, UnicodeString & FileName, TObject *& Object, UnicodeString & TargetDir, bool & Dir,
bool & Recursed, TCopyParamType *& CustomCopyParam);
void Done(
const UnicodeString & FileName, bool Dir, bool Success, const UnicodeString & TargetDir,
const TCopyParamType * CopyParam, TTerminal * Terminal);
bool UpdateFileList(TQueueFileList * UpdateFileList);
static bool GetOnlyFile(TStrings * FileList, UnicodeString & FileName, TObject *& Object);
static TCollectedFileList * GetFileList(TStrings * FileList, int Index);
static UnicodeString GetPartPrefix(const UnicodeString & FileName);
__property TOperationSide Side = { read = FSide };
__property const TCopyParamType * CopyParam = { read = FCopyParam };
__property int Params = { read = FParams };
__property UnicodeString TargetDir = { read = FTargetDir };
__property TFileOperationProgressType * MainOperationProgress = { read = FMainOperationProgress };
__property UnicodeString MainName = { read = FMainName };
__property bool IsParallelFileTransfer = { read = FIsParallelFileTransfer };
private:
struct TDirectoryData
@@ -908,6 +924,16 @@ private:
TOperationSide FSide;
UnicodeString FMainName;
int FVersion;
bool FIsParallelFileTransfer;
__int64 FParallelFileSize;
__int64 FParallelFileOffset;
int FParallelFileCount;
UnicodeString FParallelFileTargetName;
typedef std::vector<__int64> TParallelFileOffsets;
TParallelFileOffsets FParallelFileOffsets;
std::vector<bool> FParallelFileDones;
bool FParallelFileMerging;
int FParallelFileMerged;
bool CheckEnd(TCollectedFileList * Files);
TCollectedFileList * GetFileList(int Index);

View File

@@ -631,6 +631,7 @@ bool __fastcall TWebDAVFileSystem::IsCapable(int Capability) const
case fcChangePassword:
case fcTransferOut:
case fcTransferIn:
case fcParallelFileTransfers:
return false;
case fcLocking:
@@ -1814,11 +1815,7 @@ void __fastcall TWebDAVFileSystem::Sink(
if (DeleteLocalFile)
{
FILE_OPERATION_LOOP_BEGIN
{
THROWOSIFFALSE(Sysutils::DeleteFile(ApiPath(DestFullName)));
}
FILE_OPERATION_LOOP_END(FMTLOAD(DELETE_LOCAL_FILE_ERROR, (DestFullName)));
FTerminal->DoDeleteLocalFile(DestFullName);
}
}
}

View File

@@ -8951,17 +8951,7 @@ void __fastcall TCustomScpExplorerForm::DirViewGetOverlay(
TObject * Sender, TListItem * Item, WORD & Indexes)
{
TCustomDirView * DirView = reinterpret_cast<TCustomDirView *>(Sender);
UnicodeString Ext;
if (dynamic_cast<TUnixDirView *>(DirView) != NULL)
{
Ext = UnixExtractFileExt(DirView->ItemFileName(Item));
}
else
{
Ext = ExtractFileExt(DirView->ItemFileName(Item));
}
if (SameText(Ext, Configuration->PartialExt))
if (GetPartialFileExtLen(DirView->ItemFileName(Item)) > 0)
{
Indexes |= oiPartial;
}

View File

@@ -6,6 +6,7 @@
#include <System.IOUtils.hpp>
#include <Common.h>
#include <math.h>
#include <limits>
#include "Preferences.h"
#include "Custom.h"
@@ -436,11 +437,15 @@ void __fastcall TPreferencesDialog::LoadConfiguration()
SelectPuttyRegistryStorageKey(GUIConfiguration->PuttyRegistryStorageKey);
// Queue
QueueTransferLimitEdit->AsInteger = GUIConfiguration->QueueTransfersLimit;
QueueTransferLimitEdit->AsInteger = Configuration->QueueTransfersLimit;
EnableQueueByDefaultCheck->Checked = WinConfiguration->EnableQueueByDefault;
QueueAutoPopupCheck->Checked = GUIConfiguration->QueueAutoPopup;
QueueCheck->Checked = GUIConfiguration->DefaultCopyParam.Queue;
QueueParallelCheck->Checked = GUIConfiguration->DefaultCopyParam.QueueParallel;
ParallelTransferCheck->Checked = (Configuration->ParallelTransferThreshold > 0);
__int64 ParallelTransferThreshold = static_cast<__int64>(Configuration->ParallelTransferThreshold) * 1024;
ParallelTransferThresholdCombo->Text =
SizeToStr((ParallelTransferThreshold > 0) ? ParallelTransferThreshold : 100*1024*1024);
QueueNoConfirmationCheck->Checked = GUIConfiguration->DefaultCopyParam.QueueNoConfirmation;
if (!GUIConfiguration->QueueKeepDoneItems)
{
@@ -821,11 +826,21 @@ void __fastcall TPreferencesDialog::SaveConfiguration()
}
// Queue
GUIConfiguration->QueueTransfersLimit = QueueTransferLimitEdit->AsInteger;
Configuration->QueueTransfersLimit = QueueTransferLimitEdit->AsInteger;
WinConfiguration->EnableQueueByDefault = EnableQueueByDefaultCheck->Checked;
GUIConfiguration->QueueAutoPopup = QueueAutoPopupCheck->Checked;
CopyParam.Queue = QueueCheck->Checked;
CopyParam.QueueParallel = QueueParallelCheck->Checked;
__int64 ParallelTransferThreshold;
if (ParallelTransferCheck->Checked && TryStrToSize(ParallelTransferThresholdCombo->Text, ParallelTransferThreshold))
{
Configuration->ParallelTransferThreshold = std::min(ParallelTransferThreshold / 1024, static_cast<__int64>(std::numeric_limits<int>::max()));
}
else
{
// In future, we might set it to 0, if -1 will actually mean some default threshold
Configuration->ParallelTransferThreshold = -1;
}
CopyParam.QueueNoConfirmation = QueueNoConfirmationCheck->Checked;
GUIConfiguration->QueueKeepDoneItems = (QueueKeepDoneItemsForCombo->ItemIndex != 0);
switch (QueueKeepDoneItemsForCombo->ItemIndex)
@@ -1460,6 +1475,11 @@ void __fastcall TPreferencesDialog::UpdateControls()
InterfaceChangeLabel->Visible =
!CustomWinConfiguration->CanApplyInterfaceImmediately &&
(GetInterface() != CustomWinConfiguration->AppliedInterface);
// background
EnableControl(ParallelTransferCheck, QueueParallelCheck->Checked);
EnableControl(ParallelTransferThresholdCombo, ParallelTransferCheck->Enabled && ParallelTransferCheck->Checked);
EnableControl(ParallelTransferThresholdUnitLabel, ParallelTransferThresholdCombo->Enabled);
}
}
//---------------------------------------------------------------------------
@@ -2987,19 +3007,20 @@ void __fastcall TPreferencesDialog::LanguagesViewCustomDrawItem(
}
}
//---------------------------------------------------------------------------
void __fastcall TPreferencesDialog::LogMaxSizeComboExit(TObject * /*Sender*/)
void __fastcall TPreferencesDialog::SizeComboExit(TObject * Sender)
{
__int64 Size;
if (!IsCancelButtonBeingClicked(this))
{
if (!TryStrToSize(LogMaxSizeCombo->Text, Size))
TComboBox * ComboBox = DebugNotNull(dynamic_cast<TComboBox *>(Sender));
if (!TryStrToSize(ComboBox->Text, Size))
{
LogMaxSizeCombo->SetFocus();
throw Exception(FMTLOAD(SIZE_INVALID, (LogMaxSizeCombo->Text)));
ComboBox->SetFocus();
throw Exception(FMTLOAD(SIZE_INVALID, (ComboBox->Text)));
}
else
{
LogMaxSizeCombo->Text = SizeToStr(Size);
ComboBox->Text = SizeToStr(Size);
}
}
}

View File

@@ -421,9 +421,10 @@ object PreferencesDialog: TPreferencesDialog
Width = 119
Height = 21
Anchors = [akTop, akRight]
MaxLength = 20
TabOrder = 6
OnChange = ControlChange
OnExit = LogMaxSizeComboExit
OnExit = SizeComboExit
Items.Strings = (
'1M'
'10M'
@@ -431,9 +432,9 @@ object PreferencesDialog: TPreferencesDialog
'1G')
end
object LogMaxSizeCountCheck: TCheckBox
Left = 64
Left = 56
Top = 142
Width = 186
Width = 194
Height = 17
Anchors = [akLeft, akTop, akRight]
Caption = '&Delete old log files, keep'
@@ -1531,13 +1532,13 @@ object PreferencesDialog: TPreferencesDialog
Left = 8
Top = 8
Width = 389
Height = 200
Height = 251
Anchors = [akLeft, akTop, akRight]
Caption = 'Background transfers'
TabOrder = 0
DesignSize = (
389
200)
251)
object Label5: TLabel
Left = 16
Top = 25
@@ -1548,13 +1549,22 @@ object PreferencesDialog: TPreferencesDialog
end
object QueueKeepDoneItemsCheck: TLabel
Left = 16
Top = 172
Top = 223
Width = 198
Height = 13
Caption = 'Display &completed transfers in queue for:'
FocusControl = QueueKeepDoneItemsForCombo
OnClick = ControlChange
end
object ParallelTransferThresholdUnitLabel: TLabel
Left = 147
Top = 148
Width = 27
Height = 13
Caption = 'bytes'
FocusControl = ParallelTransferThresholdCombo
OnClick = ControlChange
end
object QueueTransferLimitEdit: TUpDownEdit
Left = 304
Top = 22
@@ -1569,12 +1579,12 @@ object PreferencesDialog: TPreferencesDialog
end
object QueueAutoPopupCheck: TCheckBox
Left = 16
Top = 146
Top = 197
Width = 369
Height = 17
Anchors = [akLeft, akTop, akRight]
Caption = '&Automatically popup prompts of background transfers when idle'
TabOrder = 5
TabOrder = 7
end
object QueueCheck: TCheckBox
Left = 16
@@ -1587,12 +1597,12 @@ object PreferencesDialog: TPreferencesDialog
end
object QueueNoConfirmationCheck: TCheckBox
Left = 16
Top = 122
Top = 173
Width = 369
Height = 17
Anchors = [akLeft, akTop, akRight]
Caption = '&No confirmations for background transfers'
TabOrder = 4
TabOrder = 6
end
object QueueParallelCheck: TCheckBox
Left = 16
@@ -1602,6 +1612,7 @@ object PreferencesDialog: TPreferencesDialog
Anchors = [akLeft, akTop, akRight]
Caption = '&Use multiple connections for single transfer'
TabOrder = 3
OnClick = ControlChange
end
object EnableQueueByDefaultCheck: TCheckBox
Left = 16
@@ -1614,13 +1625,13 @@ object PreferencesDialog: TPreferencesDialog
end
object QueueKeepDoneItemsForCombo: TComboBox
Left = 280
Top = 169
Top = 220
Width = 97
Height = 21
Style = csDropDownList
Anchors = [akTop, akRight]
MaxLength = 1
TabOrder = 6
TabOrder = 8
OnChange = ControlChange
Items.Strings = (
'Never'
@@ -1630,10 +1641,36 @@ object PreferencesDialog: TPreferencesDialog
'1 hour'
'Forever')
end
object ParallelTransferCheck: TCheckBox
Left = 32
Top = 122
Width = 345
Height = 17
Anchors = [akLeft, akTop, akRight]
Caption = '&Use multiple connections for single files abo&ve:'
TabOrder = 4
OnClick = ControlChange
end
object ParallelTransferThresholdCombo: TComboBox
Left = 48
Top = 145
Width = 93
Height = 21
Anchors = [akTop, akRight]
MaxLength = 20
TabOrder = 5
OnChange = ControlChange
OnExit = SizeComboExit
Items.Strings = (
'1M'
'10M'
'100M'
'1G')
end
end
object QueueViewGroup: TGroupBox
Left = 8
Top = 214
Top = 265
Width = 389
Height = 99
Anchors = [akLeft, akTop, akRight]

View File

@@ -344,6 +344,9 @@ __published:
TLabel *LogProtocolHintLabel;
TGroupBox *EditingOptionsGroup;
TCheckBox *EditorCheckNotModifiedCheck;
TCheckBox *ParallelTransferCheck;
TComboBox *ParallelTransferThresholdCombo;
TLabel *ParallelTransferThresholdUnitLabel;
void __fastcall FormShow(TObject *Sender);
void __fastcall ControlChange(TObject *Sender);
void __fastcall EditorFontButtonClick(TObject *Sender);
@@ -433,7 +436,7 @@ __published:
void __fastcall BackgroundConfirmationsLinkClick(TObject *Sender);
void __fastcall ConfigureCommandButtonClick(TObject *Sender);
void __fastcall LanguagesViewCustomDrawItem(TCustomListView * Sender, TListItem * Item, TCustomDrawState State, bool & DefaultDraw);
void __fastcall LogMaxSizeComboExit(TObject *Sender);
void __fastcall SizeComboExit(TObject *Sender);
void __fastcall PuttyPathEditExit(TObject *Sender);
void __fastcall AutomaticIniFileStorageLabelGetStatus(TCustomPathLabel *Sender, bool &Active);
void __fastcall CustomIniFileStorageEditExit(TObject *Sender);

View File

@@ -2120,10 +2120,10 @@ void __fastcall TScpCommanderForm::RemotePathLabelPathClick(
void __fastcall TScpCommanderForm::LocalDirViewFileIconForName(
TObject * /*Sender*/, UnicodeString & FileName)
{
UnicodeString PartialExt = Configuration->PartialExt;
if (AnsiSameText(ExtractFileExt(FileName), PartialExt))
int PartialFileExtLen = GetPartialFileExtLen(FileName);
if (PartialFileExtLen > 0)
{
FileName.SetLength(FileName.Length() - PartialExt.Length());
FileName.SetLength(FileName.Length() - PartialFileExtLen);
}
if (WinConfiguration->LocalIconsByExt)
{

View File

@@ -286,6 +286,7 @@
#define CERTIFICATE_NOT_MATCH 762
#define CERTIFICATE_CANNOT_COMBINE 763
#define KEYGEN_NOT_PUBLIC 764
#define INCONSISTENT_SIZE 765
#define CORE_CONFIRMATION_STRINGS 300
#define CONFIRM_PROLONG_TIMEOUT3 301

View File

@@ -258,6 +258,7 @@ BEGIN
CERTIFICATE_NOT_MATCH, "Certificate in \"%s\" does not match public key in the key file."
CERTIFICATE_CANNOT_COMBINE, "Unable to combine certificate in \"%s\" with private key."
KEYGEN_NOT_PUBLIC, "File \"%s\" is not a public key in a known format."
INCONSISTENT_SIZE, "File part \"%s\" size is %s, but %s was expected."
CORE_CONFIRMATION_STRINGS, "CORE_CONFIRMATION"
CONFIRM_PROLONG_TIMEOUT3, "Host is not communicating for %d seconds.\n\nWait for another %0:d seconds?"

View File

@@ -556,7 +556,6 @@ void __fastcall TGUIConfiguration::Default()
FSynchronizeMode = TTerminal::smRemote;
FMaxWatchDirectories = 500;
FSynchronizeOptions = soRecurse | soSynchronizeAsk;
FQueueTransfersLimit = 2;
FQueueBootstrap = false;
FQueueKeepDoneItems = true;
FQueueKeepDoneItemsFor = 15;
@@ -636,7 +635,6 @@ void __fastcall TGUIConfiguration::UpdateStaticUsage()
KEY(Integer, SynchronizeModeAuto); \
KEY(Integer, SynchronizeMode); \
KEY(Integer, MaxWatchDirectories); \
KEY(Integer, QueueTransfersLimit); \
KEY(Bool, QueueBootstrap); \
KEY(Integer, QueueKeepDoneItems); \
KEY(Integer, QueueKeepDoneItemsFor); \
@@ -1350,11 +1348,6 @@ void __fastcall TGUIConfiguration::SetNewDirectoryProperties(
SET_CONFIG_PROPERTY(NewDirectoryProperties);
}
//---------------------------------------------------------------------------
void __fastcall TGUIConfiguration::SetQueueTransfersLimit(int value)
{
SET_CONFIG_PROPERTY(QueueTransfersLimit);
}
//---------------------------------------------------------------------------
void __fastcall TGUIConfiguration::SetQueueBootstrap(bool value)
{
SET_CONFIG_PROPERTY(QueueBootstrap);

View File

@@ -170,7 +170,6 @@ private:
TDateTime FIgnoreCancelBeforeFinish;
bool FQueueAutoPopup;
bool FSessionRememberPassword;
int FQueueTransfersLimit;
bool FQueueBootstrap;
bool FQueueKeepDoneItems;
int FQueueKeepDoneItemsFor;
@@ -220,7 +219,6 @@ protected:
void __fastcall SetCopyParamCurrent(UnicodeString value);
void __fastcall SetNewDirectoryProperties(const TRemoteProperties & value);
virtual void __fastcall Saved();
void __fastcall SetQueueTransfersLimit(int value);
void __fastcall SetQueueBootstrap(bool value);
void __fastcall SetQueueKeepDoneItems(bool value);
void __fastcall SetQueueKeepDoneItemsFor(int value);
@@ -260,7 +258,6 @@ public:
__property int SynchronizeModeAuto = { read = FSynchronizeModeAuto, write = FSynchronizeModeAuto };
__property int SynchronizeMode = { read = FSynchronizeMode, write = FSynchronizeMode };
__property int MaxWatchDirectories = { read = FMaxWatchDirectories, write = FMaxWatchDirectories };
__property int QueueTransfersLimit = { read = FQueueTransfersLimit, write = SetQueueTransfersLimit };
__property bool QueueBootstrap = { read = FQueueBootstrap, write = SetQueueBootstrap };
__property bool QueueKeepDoneItems = { read = FQueueKeepDoneItems, write = SetQueueKeepDoneItems };
__property int QueueKeepDoneItemsFor = { read = FQueueKeepDoneItemsFor, write = SetQueueKeepDoneItemsFor };

View File

@@ -131,7 +131,7 @@ __fastcall TTerminalManager::~TTerminalManager()
//---------------------------------------------------------------------------
void __fastcall TTerminalManager::SetQueueConfiguration(TTerminalQueue * Queue)
{
Queue->TransfersLimit = GUIConfiguration->QueueTransfersLimit;
Queue->TransfersLimit = Configuration->QueueTransfersLimit;
Queue->KeepDoneItemsFor =
(GUIConfiguration->QueueKeepDoneItems ? GUIConfiguration->QueueKeepDoneItemsFor : 0);
}