sysutils/winscp
1
0
Fork 0
mirror of https://github.com/winscp/winscp.git synced 2026-10-24 02:28:55 +03:00
Code Activity

Bug 1945: Streaming support in .NET assembly for FTP protocol

Browse Source 
https://winscp.net/tracker/1945

Source commit: 01274024eff31ef3f5794f3521689b4c2229acbb
This commit is contained in:
Martin Prikryl
2022-04-29 13:40:41 +00:00
parent c9ff66a226
commit 9370bfd17f
8 changed files with 142 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
source/core/FtpFileSystem.cpp
View File

@@ -1581,8 +1581,9 @@ void __fastcall TFTPFileSystem::FileTransfer(const UnicodeString & FileName,
{
FILE_OPERATION_LOOP_BEGIN
{
FFileZillaIntf->FileTransfer(ApiPath(LocalFile).c_str(), RemoteFile.c_str(),
RemotePath.c_str(), Get, Size, Type, &UserData);
FFileZillaIntf->FileTransfer(
ApiPath(LocalFile).c_str(), RemoteFile.c_str(), RemotePath.c_str(),
Get, Size, Type, &UserData, UserData.CopyParam->OnTransferOut, UserData.CopyParam->OnTransferIn);
// we may actually catch response code of the listing
// command (when checking for existence of the remote file)
unsigned int Reply = WaitForCommandReply();
@@ -1694,7 +1695,10 @@ void __fastcall TFTPFileSystem::Sink(
UnicodeString ExpandedDestFullName = ExpandUNCFileName(DestFullName);
Action.Destination(ExpandedDestFullName);
FTerminal->UpdateTargetAttrs(DestFullName, File, CopyParam, Attrs);
if (CopyParam->OnTransferOut == NULL)
{
FTerminal->UpdateTargetAttrs(DestFullName, File, CopyParam, Attrs);
}
FLastDataSent = Now();
}
@@ -1718,12 +1722,13 @@ void __fastcall TFTPFileSystem::CopyToRemote(TStrings * FilesToCopy,
bool __fastcall TFTPFileSystem::CanTransferSkipList(int Params, unsigned int Flags, const TCopyParamType * CopyParam)
{
bool Result =
FLAGSET(Params, cpNoConfirmation) &&
// cpAppend is not supported with FTP
DebugAlwaysTrue(FLAGCLEAR(Params, cpAppend)) &&
FLAGCLEAR(Params, cpResume) &&
FLAGCLEAR(Flags, tfAutoResume) &&
!CopyParam->NewerOnly;
(CopyParam->OnTransferIn != NULL) ||
(FLAGSET(Params, cpNoConfirmation) &&
// cpAppend is not supported with FTP
DebugAlwaysTrue(FLAGCLEAR(Params, cpAppend)) &&
FLAGCLEAR(Params, cpResume) &&
FLAGCLEAR(Flags, tfAutoResume) &&
!CopyParam->NewerOnly);
return Result;
}
//---------------------------------------------------------------------------
@@ -1733,7 +1738,10 @@ void __fastcall TFTPFileSystem::Source(
TFileOperationProgressType * OperationProgress, unsigned int Flags,
TUploadSessionAction & Action, bool & /*ChildError*/)
{
Handle.Close();
if (CopyParam->OnTransferIn == NULL)
{
Handle.Close();
}
ResetFileTransfer();
@@ -1749,8 +1757,9 @@ void __fastcall TFTPFileSystem::Source(
SetCPSLimit(OperationProgress);
// not used for uploads anyway
FFileTransferPreserveTime = CopyParam->PreserveTime;
FFileTransferPreserveTime = CopyParam->PreserveTime && (CopyParam->OnTransferIn == NULL);
FFileTransferRemoveBOM = CopyParam->RemoveBOM;
// not used for uploads anyway
FFileTransferNoList = CanTransferSkipList(Params, Flags, CopyParam);
// not used for uploads, but we get new name (if any) back in this field
UserData.FileName = DestFileName;
@@ -1927,6 +1936,8 @@ bool __fastcall TFTPFileSystem::IsCapable(int Capability) const
case fcMoveToQueue:
case fcSkipTransfer:
case fcParallelTransfers:
case fcTransferOut:
case fcTransferIn:
return true;
case fcPreservingTimestampUpload:
@@ -1962,8 +1973,6 @@ bool __fastcall TFTPFileSystem::IsCapable(int Capability) const
case fcPreservingTimestampDirs:
case fcResumeSupport:
case fcChangePassword:
case fcTransferOut:
case fcTransferIn:
return false;
default:
@@ -3877,6 +3886,11 @@ bool __fastcall TFTPFileSystem::HandleAsynchRequestOverwrite(
// on retry, use the same answer as on the first attempt
RequestResult = UserData.OverwriteResult;
}
else if ((UserData.CopyParam->OnTransferOut != NULL) || (UserData.CopyParam->OnTransferIn != NULL))
{
DebugFail();
RequestResult = TFileZillaIntf::FILEEXISTS_OVERWRITE;
}
else
{
TFileOperationProgressType * OperationProgress = FTerminal->OperationProgress;
@@ -4684,6 +4698,7 @@ bool __fastcall TFTPFileSystem::Unquote(UnicodeString & Str)
void __fastcall TFTPFileSystem::PreserveDownloadFileTime(HANDLE Handle, void * UserData)
{
TFileTransferData * Data = static_cast<TFileTransferData *>(UserData);
DebugAssert(Data->CopyParam->OnTransferOut == NULL);
FTerminal->UpdateTargetTime(Handle, Data->Modification, dstmUnix);
}
//---------------------------------------------------------------------------

9
source/filezilla/FileZillaIntf.cpp
View File

@@ -218,9 +218,10 @@ bool __fastcall TFileZillaIntf::ListFile(const wchar_t * FileName, const wchar_t
return Check(FFileZillaApi->ListFile(FileName, Path), L"listfile");
}
//---------------------------------------------------------------------------
bool __fastcall TFileZillaIntf::FileTransfer(const wchar_t * LocalFile,
const wchar_t * RemoteFile, const wchar_t * RemotePath, bool Get, __int64 Size,
int Type, void * UserData)
bool __fastcall TFileZillaIntf::FileTransfer(
const wchar_t * LocalFile, const wchar_t * RemoteFile,
const wchar_t * RemotePath, bool Get, __int64 Size, int Type, void * UserData,
TTransferOutEvent OnTransferOut, TTransferInEvent OnTransferIn)
{
t_transferfile Transfer;
@@ -233,6 +234,8 @@ bool __fastcall TFileZillaIntf::FileTransfer(const wchar_t * LocalFile,
// 1 = ascii, 2 = binary
Transfer.nType = Type;
Transfer.nUserData = reinterpret_cast<int>(UserData);
Transfer.OnTransferOut = OnTransferOut;
Transfer.OnTransferIn = OnTransferIn;
return Check(FFileZillaApi->FileTransfer(Transfer), L"filetransfer");
}

6
source/filezilla/FileZillaIntf.h
View File

@@ -195,8 +195,10 @@ public:
bool __fastcall Rename(const wchar_t* OldName, const wchar_t* NewName,
const wchar_t* Path, const wchar_t* NewPath);
bool __fastcall FileTransfer(const wchar_t * LocalFile, const wchar_t * RemoteFile,
const wchar_t * RemotePath, bool Get, __int64 Size, int Type, void * UserData);
bool __fastcall FileTransfer(
const wchar_t * LocalFile, const wchar_t * RemoteFile,
const wchar_t * RemotePath, bool Get, __int64 Size, int Type, void * UserData,
TTransferOutEvent OnTransferOut, TTransferInEvent OnTransferIn);
virtual const wchar_t * __fastcall Option(int OptionID) const = 0;
virtual int __fastcall OptionVal(int OptionID) const = 0;

71
source/filezilla/FtpControlSocket.cpp
View File

@@ -2841,10 +2841,14 @@ void CFtpControlSocket::FileTransfer(t_transferfile *transferfile/*=0*/,BOOL bFi
DebugAssert(!m_Operation.nOpMode);
DebugAssert(!m_Operation.pData);
CString str;
str.Format(transferfile->get?IDS_STATUSMSG_DOWNLOADSTART:IDS_STATUSMSG_UPLOADSTART,
transferfile->get ? transferfile->remotepath.FormatFilename(transferfile->remotefile) : transferfile->localfile);
ShowStatus(str,FZ_LOG_STATUS);
if ((transferfile->OnTransferOut == NULL) &&
(transferfile->OnTransferIn == NULL))
{
CString str;
str.Format(transferfile->get?IDS_STATUSMSG_DOWNLOADSTART:IDS_STATUSMSG_UPLOADSTART,
transferfile->get ? transferfile->remotepath.FormatFilename(transferfile->remotefile) : transferfile->localfile);
ShowStatus(str,FZ_LOG_STATUS);
}
m_Operation.nOpMode=CSMODE_TRANSFER|(transferfile->get?CSMODE_DOWNLOAD:CSMODE_UPLOAD);
@@ -3502,18 +3506,31 @@ void CFtpControlSocket::FileTransfer(t_transferfile *transferfile/*=0*/,BOOL bFi
m_Operation.nOpState = FILETRANSFER_REST;
else
m_Operation.nOpState = FILETRANSFER_RETRSTOR;
BOOL res = FALSE;
if (!m_pDataFile)
m_pDataFile = new CFile;
BOOL res;
if (m_pDataFile != NULL)
{
delete m_pDataFile;
m_pDataFile = NULL;
}
if (pData->transferfile.get)
{
if (pData->transferdata.bResume)
res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeCreate|CFile::modeWrite|CFile::modeNoTruncate|CFile::shareDenyWrite);
else
res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeWrite|CFile::modeCreate|CFile::shareDenyWrite);
if (pData->transferfile.OnTransferOut == NULL)
{
m_pDataFile = new CFile();
if (pData->transferdata.bResume)
res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeCreate|CFile::modeWrite|CFile::modeNoTruncate|CFile::shareDenyWrite);
else
res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeWrite|CFile::modeCreate|CFile::shareDenyWrite);
}
}
else
res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeRead|CFile::shareDenyNone);
{
if (pData->transferfile.OnTransferIn == NULL)
{
m_pDataFile = new CFile();
res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeRead|CFile::shareDenyNone);
}
}
if (!res)
{
wchar_t * Error = m_pTools->LastSysErrorMessage();
@@ -3535,11 +3552,16 @@ void CFtpControlSocket::FileTransfer(t_transferfile *transferfile/*=0*/,BOOL bFi
}
m_pTransferSocket->m_pFile = m_pDataFile;
m_pTransferSocket->m_OnTransferOut = pData->transferfile.OnTransferOut;
m_pTransferSocket->m_OnTransferIn = pData->transferfile.OnTransferIn;
if (!pData->transferfile.get)
{
// See comment in !get branch below
pData->transferdata.transfersize=GetLength64(*m_pDataFile);
pData->transferdata.transferleft=pData->transferdata.transfersize;
if (m_pDataFile != NULL)
{
// See comment in !get branch below
pData->transferdata.transfersize=GetLength64(*m_pDataFile);
pData->transferdata.transferleft=pData->transferdata.transfersize;
}
if (pData->transferdata.bResume)
{
CString remotefile=pData->transferfile.remotefile;
@@ -4385,8 +4407,10 @@ void CFtpControlSocket::TransferFinished(bool preserveFileTimeForUploads)
{
CFileTransferData *pData=static_cast<CFileTransferData *>(m_Operation.pData);
if (GetOptionVal(OPTION_PRESERVEDOWNLOADFILETIME) && m_pDataFile &&
pData->transferfile.get)
if (GetOptionVal(OPTION_PRESERVEDOWNLOADFILETIME) &&
m_pDataFile &&
pData->transferfile.get &&
DebugAlwaysTrue(pData->transferfile.OnTransferOut == NULL))
{
m_pTools->PreserveDownloadFileTime(
(HANDLE)m_pDataFile->m_hFile, reinterpret_cast<void *>(pData->transferfile.nUserData));
@@ -4394,7 +4418,8 @@ void CFtpControlSocket::TransferFinished(bool preserveFileTimeForUploads)
if (!pData->transferfile.get &&
GetOptionVal(OPTION_MPEXT_PRESERVEUPLOADFILETIME) && preserveFileTimeForUploads &&
((m_serverCapabilities.GetCapability(mfmt_command) == yes) ||
(m_serverCapabilities.GetCapability(mdtm_command) == yes)))
(m_serverCapabilities.GetCapability(mdtm_command) == yes)) &&
DebugAlwaysTrue(pData->transferfile.OnTransferIn == NULL))
{
CString filename =
pData->transferfile.remotepath.FormatFilename(pData->transferfile.remotefile, !pData->bUseAbsolutePaths);
@@ -4867,7 +4892,7 @@ int CFtpControlSocket::CheckOverwriteFileAndCreateTarget()
if (!nReplyError)
{
CFileTransferData * pData = static_cast<CFileTransferData *>(m_Operation.pData);
if (pData->transferfile.get)
if (pData->transferfile.get && (pData->transferfile.OnTransferOut == NULL))
{
CString path = pData->transferfile.localfile;
if (path.ReverseFind(L'\\') != -1)
@@ -4897,8 +4922,12 @@ int CFtpControlSocket::CheckOverwriteFile()
int nReplyError = 0;
CFileStatus64 status;
BOOL res = GetStatus64(pData->transferfile.localfile, status);
if (!res)
if ((pData->transferfile.OnTransferOut != NULL) ||
(pData->transferfile.OnTransferIn != NULL))
{
m_Operation.nOpState = FILETRANSFER_TYPE;
}
else if (!GetStatus64(pData->transferfile.localfile, status))
{
if (!pData->transferfile.get)
{

3
source/filezilla/FzApiStructures.h
View File

@@ -3,6 +3,7 @@
#define FzApiStructuresH
//---------------------------------------------------------------------------
#include <openssl/pkcs12.h>
#include <FileBuffer.h>
//---------------------------------------------------------------------------
class t_server
{
@@ -35,6 +36,8 @@ typedef struct
t_server server;
int nType;
int nUserData;
TTransferOutEvent OnTransferOut;
TTransferInEvent OnTransferIn;
} t_transferfile;
//---------------------------------------------------------------------------
#endif // FzApiStructuresH

41
source/filezilla/TransferSocket.cpp
View File

@@ -30,6 +30,8 @@ CTransferSocket::CTransferSocket(CFtpControlSocket *pOwner, int nMode)
#endif
m_bufferpos = 0;
m_pFile = 0;
m_OnTransferOut = NULL;
m_OnTransferIn = NULL;
m_bListening = FALSE;
m_bSentClose = FALSE;
m_nInternalMessageID = 0;
@@ -256,7 +258,7 @@ void CTransferSocket::OnReceive(int nErrorCode)
int res = inflate(&m_zlibStream, 0);
while (res == Z_OK)
{
m_pFile->Write(m_pBuffer2, BUFSIZE - m_zlibStream.avail_out);
WriteData(m_pBuffer2, BUFSIZE - m_zlibStream.avail_out);
written += BUFSIZE - m_zlibStream.avail_out;
m_zlibStream.next_out = (Bytef *)m_pBuffer2;
m_zlibStream.avail_out = BUFSIZE;
@@ -264,7 +266,7 @@ void CTransferSocket::OnReceive(int nErrorCode)
}
if (res == Z_STREAM_END)
{
m_pFile->Write(m_pBuffer2, BUFSIZE - m_zlibStream.avail_out);
WriteData(m_pBuffer2, BUFSIZE - m_zlibStream.avail_out);
written += BUFSIZE - m_zlibStream.avail_out;
}
else if (res != Z_OK && res != Z_BUF_ERROR)
@@ -277,7 +279,7 @@ void CTransferSocket::OnReceive(int nErrorCode)
else
#endif
{
m_pFile->Write(m_pBuffer, numread);
WriteData(m_pBuffer, numread);
written = numread;
}
}
@@ -689,7 +691,7 @@ void CTransferSocket::OnSend(int nErrorCode)
else
#endif
{
if (!m_pFile)
if (!m_pFile && (m_OnTransferIn == NULL))
{
return;
}
@@ -1098,6 +1100,33 @@ bool CTransferSocket::InitZlib(int level)
}
#endif
void CTransferSocket::WriteData(const char * buffer, int len)
{
if (m_OnTransferOut != NULL)
{
m_OnTransferOut(NULL, m_pBuffer, len);
}
else
{
m_pFile->Write(m_pBuffer, len);
}
}
int CTransferSocket::ReadData(char * buffer, int len)
{
int result;
if (m_OnTransferIn != NULL)
{
result = m_OnTransferIn(NULL, buffer, len);
}
else
{
result = m_pFile->Read(buffer, len);
}
LogMessage(FZ_LOG_INFO, L"Read %d bytes from file", result);
return result;
}
int CTransferSocket::ReadDataFromFile(char *buffer, int len)
{
TRY
@@ -1105,13 +1134,13 @@ int CTransferSocket::ReadDataFromFile(char *buffer, int len)
// Comparing to Filezilla 2, we do not do any translation locally,
// leaving it onto the server (what Filezilla 3 seems to do too)
const char Bom[3] = "\xEF\xBB\xBF";
int read = m_pFile->Read(buffer, len);
int read = ReadData(buffer, len);
if (GetOptionVal(OPTION_MPEXT_REMOVE_BOM) &&
m_transferdata.bType && (read >= sizeof(Bom)) && (memcmp(buffer, Bom, sizeof(Bom)) == 0))
{
memcpy(buffer, buffer + sizeof(Bom), read - sizeof(Bom));
read -= sizeof(Bom);
int read2 = m_pFile->Read(buffer + read, sizeof(Bom));
int read2 = ReadData(buffer + read, sizeof(Bom));
if (read2 > 0)
{
read += read2;

4
source/filezilla/TransferSocket.h
View File

@@ -33,6 +33,8 @@ public:
virtual BOOL Create(BOOL bUseSsl);
BOOL m_bListening;
CFile * m_pFile;
TTransferOutEvent m_OnTransferOut;
TTransferInEvent m_OnTransferIn;
t_transferdata m_transferdata;
__int64 m_uploaded;
void SetActive();
@@ -55,6 +57,8 @@ public:
protected:
virtual int OnLayerCallback(std::list<t_callbackMsg> & callbacks);
int ReadDataFromFile(char * buffer, int len);
int ReadData(char * buffer, int len);
void WriteData(const char * buffer, int len);
virtual void LogSocketMessageRaw(int nMessageType, LPCTSTR pMsg);
virtual int GetSocketOptionVal(int OptionID) const;
virtual void ConfigureSocket();

12
source/windows/ConsoleRunner.cpp
View File

@@ -560,6 +560,8 @@ private:
bool FWantsProgress;
bool FInteractive;
unsigned int FMaxSend;
// Particularly FTP calls TransferOut/In from other thread
std::unique_ptr<TCriticalSection> FSection;
inline TConsoleCommStruct * __fastcall GetCommStruct();
inline void __fastcall FreeCommStruct(TConsoleCommStruct * CommStruct);
@@ -592,6 +594,8 @@ __fastcall TExternalConsole::TExternalConsole(
CloseHandle(Job);
}
FSection.reset(new TCriticalSection());
TConsoleCommStruct * CommStruct = GetCommStruct();
try
{
@@ -686,6 +690,7 @@ UnicodeString __fastcall TExternalConsole::FinalLogMessage()
//---------------------------------------------------------------------------
void __fastcall TExternalConsole::Print(UnicodeString Str, bool FromBeginning, bool Error)
{
TGuard Guard(FSection.get());
// need to do at least one iteration, even when Str is empty (new line)
do
{
@@ -721,6 +726,7 @@ void __fastcall TExternalConsole::Print(UnicodeString Str, bool FromBeginning, b
//---------------------------------------------------------------------------
bool __fastcall TExternalConsole::Input(UnicodeString & Str, bool Echo, unsigned int Timer)
{
TGuard Guard(FSection.get());
TConsoleCommStruct * CommStruct = GetCommStruct();
try
{
@@ -757,6 +763,7 @@ int __fastcall TExternalConsole::Choice(
UnicodeString Options, int Cancel, int Break, int Continue, int Timeouted, bool Timeouting, unsigned int Timer,
UnicodeString Message)
{
TGuard Guard(FSection.get());
TConsoleCommStruct * CommStruct = GetCommStruct();
try
{
@@ -803,6 +810,7 @@ bool __fastcall TExternalConsole::PendingAbort()
//---------------------------------------------------------------------------
void __fastcall TExternalConsole::SetTitle(UnicodeString Title)
{
TGuard Guard(FSection.get());
TConsoleCommStruct * CommStruct = GetCommStruct();
try
{
@@ -822,6 +830,7 @@ void __fastcall TExternalConsole::SetTitle(UnicodeString Title)
//---------------------------------------------------------------------------
void __fastcall TExternalConsole::Init()
{
TGuard Guard(FSection.get());
TConsoleCommStruct * CommStruct = GetCommStruct();
try
{
@@ -897,6 +906,7 @@ void __fastcall TExternalConsole::WaitBeforeExit()
//---------------------------------------------------------------------------
void __fastcall TExternalConsole::Progress(TScriptProgress & Progress)
{
TGuard Guard(FSection.get());
TConsoleCommStruct * CommStruct = GetCommStruct();
typedef TConsoleCommStruct::TProgressEvent TProgressEvent;
@@ -957,6 +967,7 @@ void __fastcall TExternalConsole::Progress(TScriptProgress & Progress)
//---------------------------------------------------------------------------
void __fastcall TExternalConsole::TransferOut(const unsigned char * Data, size_t Len)
{
TGuard Guard(FSection.get());
DebugAssert((Data == NULL) == (Len == 0));
size_t Offset = 0;
do
@@ -981,6 +992,7 @@ void __fastcall TExternalConsole::TransferOut(const unsigned char * Data, size_t
//---------------------------------------------------------------------------
size_t __fastcall TExternalConsole::TransferIn(unsigned char * Data, size_t Len)
{
TGuard Guard(FSection.get());
size_t Offset = 0;
size_t Result = 0;
while ((Result == Offset) && (Offset < Len))