database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-07-30 19:03:16 +03:00
Code Activity

Enhancements to the ".import" command of the CLI.

Browse Source 
FossilOrigin-Name: cab1834cfc71f71bfed3c5170a0ba40a39385c3b2c50b7c6b6f09cc830dd1b1e
This commit is contained in:
drh
2020-03-09 15:39:39 +00:00
parent 45dc9ca46b
commit ccb3781673
5 changed files with 161 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
manifest
View File

@ -1,5 +1,5 @@
C Fix\stypos\sin\sRowSet.
D 2020-03-09T03:21:33.427
C Enhancements\sto\sthe\s".import"\scommand\sof\sthe\sCLI.
D 2020-03-09T15:39:39.029
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -532,7 +532,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
F src/resolve.c 38e3a5636f5bdc92e3683e4cafbba6418c0aa15e0d89ca5b28bd0b621dbb80bf
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c c94eec317c8ba929bc228392eb3cac8124f2d0fbe3fc1bddecb44dfc7057bc78
F src/shell.c.in 3897f3f7302914da1f6df3a2a09ac4aafa14a571d7d18c51500cfb2ff04f05eb
F src/shell.c.in f76590931c0cbbfef347f44f81ade6b335f80c46bc6e59b8b6114383a8df30e0
F src/sqlite.h.in 802957feeb249ede54f8dfe99b72aa19e70a0b7737969c46e625dc2f9f2d42b0
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 9c5269260409eb3275324ccace6a13a96f4ad330c708415f70ca6097901ff4ee
@ -1333,11 +1333,11 @@ F test/sharedA.test 49d87ec54ab640fbbc3786ee3c01de94aaa482a3a9f834ad3fe92770eb69
F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e
F test/shared_err.test 32634e404a3317eeb94abc7a099c556a346fdb8fb3858dbe222a4cbb8926a939
F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304
F test/shell1.test 3c9707dce15e8fdca529503378660f099777d3ddcedccf801a37589a405c5942
F test/shell1.test 43e12c7d4ff65f041803ad24a93fd3818deb2cb83e721810f27d0fde61d64a13
F test/shell2.test e242a9912f44f4c23c3d1d802a83e934e84c853b
F test/shell3.test ac8c2b744014c3e9a0e26bfd829ab65f00923dc1a91ffd044863e9423cc91494
F test/shell4.test 1c6aef11daaa2d6830acaba3ac9cbec93fbc1c3d5530743a637f39b3987d08ce
F test/shell5.test 23939a4c51f0421330ea61dbd3c74f9c215f5f8d3d1a94846da6ffc777a35458
F test/shell5.test 84a30b55722a95a5b72989e691c469a999ca7591e7aa00b7fabc783ea5c9a6fe
F test/shell6.test 1ceb51b2678c472ba6cf1e5da96679ce8347889fe2c3bf93a0e0fa73f00b00d3
F test/shell7.test 115132f66d0463417f408562cc2cf534f6bbc6d83a6d50f0072a9eb171bae97f
F test/shell8.test 96be02ea0c21f05b24c1883d7b711a1fa8525a68ab7b636aacf6057876941013
@ -1860,7 +1860,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 35f1f151ac478d6b46f3685d2565c35108ef74bd33ce96fb65300d3c303b289b
R 3ad66a51f32de0dab569042c4aa4c3c6
U pdr
Z ab82beefd8209a83b9f274edf5605106
P 86465c08f4d629a296332a7985937326ac43ea2822c5651bf03862cd79d370fc
R 59ca84b4f9c863c565a73d91cc01fad5
U drh
Z ca4df02a4f4403d62ad30a31c55381b8

2
manifest.uuid
View File

@ -1 +1 @@
86465c08f4d629a296332a7985937326ac43ea2822c5651bf03862cd79d370fc
cab1834cfc71f71bfed3c5170a0ba40a39385c3b2c50b7c6b6f09cc830dd1b1e

192
src/shell.c.in
View File

@ -3575,6 +3575,18 @@ static const char *(azHelp[]) = {
".headers on|off Turn display of headers on or off",
".help ?-all? ?PATTERN? Show help text for PATTERN",
".import FILE TABLE Import data from FILE into TABLE",
" Options:",
" --ascii Use \\037 and \\036 as column and row separators",
" --csv Use , and \\n as column and row separators",
" --skip N Skip the first N rows of input",
" -v \"Verbose\" - increase auxiliary output",
" Notes:",
" * If TABLE does not exist, it is created. The first row of input",
" determines the column names.",
" * If neither --csv or --ascii are used, the input mode is derived",
" from the \".mode\" output mode",
" * If FILE begins with \"|\" then it is a command that generates the",
" input text.",
#ifndef SQLITE_OMIT_TEST_CONTROL
".imposter INDEX TABLE Create imposter table TABLE on index INDEX",
#endif
@ -4563,6 +4575,8 @@ struct ImportCtx {
int n; /* Number of bytes in z */
int nAlloc; /* Space allocated for z[] */
int nLine; /* Current line number */
int nRow; /* Number of rows imported */
int nErr; /* Number of errors encountered */
int bNotFirst; /* True if one or more bytes already read */
int cTerm; /* Character that terminated the most recent field */
int cColSep; /* The column separator character. (Usually ",") */
@ -7623,8 +7637,8 @@ static int do_meta_command(char *zLine, ShellState *p){
}else
if( c=='i' && strncmp(azArg[0], "import", n)==0 ){
char *zTable; /* Insert data into this table */
char *zFile; /* Name of file to extra content from */
char *zTable = 0; /* Insert data into this table */
char *zFile = 0; /* Name of file to extra content from */
sqlite3_stmt *pStmt = NULL; /* A statement */
int nCol; /* Number of columns in the table */
int nByte; /* Number of bytes in an SQL string */
@ -7635,51 +7649,108 @@ static int do_meta_command(char *zLine, ShellState *p){
ImportCtx sCtx; /* Reader context */
char *(SQLITE_CDECL *xRead)(ImportCtx*); /* Func to read one value */
int (SQLITE_CDECL *xCloser)(FILE*); /* Func to close file */
int eVerbose = 0; /* Larger for more console output */
int nSkip = 0; /* Initial lines to skip */
int useOutputMode = 1; /* Use output mode to determine separators */
if( nArg!=3 ){
raw_printf(stderr, "Usage: .import FILE TABLE\n");
memset(&sCtx, 0, sizeof(sCtx));
if( p->mode==MODE_Ascii ){
xRead = ascii_read_one_field;
}else{
xRead = csv_read_one_field;
}
for(i=1; i<nArg; i++){
char *z = azArg[i];
if( z[0]=='-' && z[1]=='-' ) z++;
if( z[0]!='-' ){
if( zFile==0 ){
zFile = z;
}else if( zTable==0 ){
zTable = z;
}else{
utf8_printf(p->out, "ERROR: extra argument: \"%s\". Usage:\n", z);
showHelp(p->out, "import");
rc = 1;
goto meta_command_exit;
}
}else if( strcmp(z,"-v")==0 ){
eVerbose++;
}else if( strcmp(z,"-skip")==0 && i<nArg-1 ){
nSkip = integerValue(azArg[++i]);
}else if( strcmp(z,"-ascii")==0 ){
sCtx.cColSep = SEP_Unit[0];
sCtx.cRowSep = SEP_Record[0];
xRead = ascii_read_one_field;
useOutputMode = 0;
}else if( strcmp(z,"-csv")==0 ){
sCtx.cColSep = ',';
sCtx.cRowSep = '\n';
xRead = csv_read_one_field;
useOutputMode = 0;
}else{
utf8_printf(p->out, "ERROR: unknown option: \"%s\". Usage:\n", z);
showHelp(p->out, "import");
rc = 1;
goto meta_command_exit;
}
}
if( zTable==0 ){
utf8_printf(p->out, "ERROR: missing %s argument. Usage:\n",
zFile==0 ? "FILE" : "TABLE");
showHelp(p->out, "import");
rc = 1;
goto meta_command_exit;
}
zFile = azArg[1];
zTable = azArg[2];
seenInterrupt = 0;
memset(&sCtx, 0, sizeof(sCtx));
open_db(p, 0);
nSep = strlen30(p->colSeparator);
if( nSep==0 ){
raw_printf(stderr,
"Error: non-null column separator required for import\n");
return 1;
}
if( nSep>1 ){
raw_printf(stderr, "Error: multi-character column separators not allowed"
" for import\n");
return 1;
}
nSep = strlen30(p->rowSeparator);
if( nSep==0 ){
raw_printf(stderr, "Error: non-null row separator required for import\n");
return 1;
}
if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator, SEP_CrLf)==0 ){
/* When importing CSV (only), if the row separator is set to the
** default output row separator, change it to the default input
** row separator. This avoids having to maintain different input
** and output row separators. */
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
if( useOutputMode ){
/* If neither the --csv or --ascii options are specified, then set
** the column and row separator characters from the output mode. */
nSep = strlen30(p->colSeparator);
if( nSep==0 ){
raw_printf(stderr,
"Error: non-null column separator required for import\n");
rc = 1;
goto meta_command_exit;
}
if( nSep>1 ){
raw_printf(stderr,
"Error: multi-character column separators not allowed"
" for import\n");
rc = 1;
goto meta_command_exit;
}
nSep = strlen30(p->rowSeparator);
}
if( nSep>1 ){
raw_printf(stderr, "Error: multi-character row separators not allowed"
" for import\n");
return 1;
if( nSep==0 ){
raw_printf(stderr,
"Error: non-null row separator required for import\n");
rc = 1;
goto meta_command_exit;
}
if( nSep==2 && p->mode==MODE_Csv && strcmp(p->rowSeparator,SEP_CrLf)==0 ){
/* When importing CSV (only), if the row separator is set to the
** default output row separator, change it to the default input
** row separator. This avoids having to maintain different input
** and output row separators. */
sqlite3_snprintf(sizeof(p->rowSeparator), p->rowSeparator, SEP_Row);
nSep = strlen30(p->rowSeparator);
}
if( nSep>1 ){
raw_printf(stderr, "Error: multi-character row separators not allowed"
" for import\n");
rc = 1;
goto meta_command_exit;
}
sCtx.cColSep = p->colSeparator[0];
sCtx.cRowSep = p->rowSeparator[0];
}
sCtx.zFile = zFile;
sCtx.nLine = 1;
if( sCtx.zFile[0]=='|' ){
#ifdef SQLITE_OMIT_POPEN
raw_printf(stderr, "Error: pipes are not supported in this OS\n");
return 1;
rc = 1;
goto meta_command_exit;
#else
sCtx.in = popen(sCtx.zFile+1, "r");
sCtx.zFile = "<pipe>";
@ -7689,17 +7760,26 @@ static int do_meta_command(char *zLine, ShellState *p){
sCtx.in = fopen(sCtx.zFile, "rb");
xCloser = fclose;
}
if( p->mode==MODE_Ascii ){
xRead = ascii_read_one_field;
}else{
xRead = csv_read_one_field;
}
if( sCtx.in==0 ){
utf8_printf(stderr, "Error: cannot open \"%s\"\n", zFile);
return 1;
rc = 1;
goto meta_command_exit;
}
if( eVerbose>=2 || (eVerbose>=1 && useOutputMode) ){
char zSep[2];
zSep[1] = 0;
zSep[0] = sCtx.cColSep;
utf8_printf(p->out, "Column separator ");
output_c_string(p->out, zSep);
utf8_printf(p->out, ", row separator ");
zSep[0] = sCtx.cRowSep;
output_c_string(p->out, zSep);
utf8_printf(p->out, "\n");
}
while( (nSkip--)>0 ){
while( xRead(&sCtx) && sCtx.cTerm==sCtx.cColSep ){}
sCtx.nLine++;
}
sCtx.cColSep = p->colSeparator[0];
sCtx.cRowSep = p->rowSeparator[0];
zSql = sqlite3_mprintf("SELECT * FROM %s", zTable);
if( zSql==0 ){
xCloser(sCtx.in);
@ -7721,9 +7801,13 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_free(sCtx.z);
xCloser(sCtx.in);
utf8_printf(stderr,"%s: empty file\n", sCtx.zFile);
return 1;
rc = 1;
goto meta_command_exit;
}
zCreate = sqlite3_mprintf("%z\n)", zCreate);
if( eVerbose>=1 ){
utf8_printf(p->out, "%s\n", zCreate);
}
rc = sqlite3_exec(p->db, zCreate, 0, 0, 0);
sqlite3_free(zCreate);
if( rc ){
@ -7731,7 +7815,8 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_errmsg(p->db));
sqlite3_free(sCtx.z);
xCloser(sCtx.in);
return 1;
rc = 1;
goto meta_command_exit;
}
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
}
@ -7740,7 +7825,8 @@ static int do_meta_command(char *zLine, ShellState *p){
if (pStmt) sqlite3_finalize(pStmt);
utf8_printf(stderr,"Error: %s\n", sqlite3_errmsg(p->db));
xCloser(sCtx.in);
return 1;
rc = 1;
goto meta_command_exit;
}
nCol = sqlite3_column_count(pStmt);
sqlite3_finalize(pStmt);
@ -7759,13 +7845,17 @@ static int do_meta_command(char *zLine, ShellState *p){
}
zSql[j++] = ')';
zSql[j] = 0;
if( eVerbose>=2 ){
utf8_printf(p->out, "Insert using: %s\n", zSql);
}
rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0);
sqlite3_free(zSql);
if( rc ){
utf8_printf(stderr, "Error: %s\n", sqlite3_errmsg(p->db));
if (pStmt) sqlite3_finalize(pStmt);
xCloser(sCtx.in);
return 1;
rc = 1;
goto meta_command_exit;
}
needCommit = sqlite3_get_autocommit(p->db);
if( needCommit ) sqlite3_exec(p->db, "BEGIN", 0, 0, 0);
@ -7808,6 +7898,9 @@ static int do_meta_command(char *zLine, ShellState *p){
if( rc!=SQLITE_OK ){
utf8_printf(stderr, "%s:%d: INSERT failed: %s\n", sCtx.zFile,
startLine, sqlite3_errmsg(p->db));
sCtx.nErr++;
}else{
sCtx.nRow++;
}
}
}while( sCtx.cTerm!=EOF );
@ -7816,6 +7909,11 @@ static int do_meta_command(char *zLine, ShellState *p){
sqlite3_free(sCtx.z);
sqlite3_finalize(pStmt);
if( needCommit ) sqlite3_exec(p->db, "COMMIT", 0, 0, 0);
if( eVerbose>0 ){
utf8_printf(p->out,
"Added %d rows with %d errors using %d lines of input\n",
sCtx.nRow, sCtx.nErr, sCtx.nLine-1);
}
}else
#ifndef SQLITE_UNTESTABLE

9
test/shell1.test
View File

@ -388,17 +388,14 @@ do_test shell1-3.10.2 {
# .import FILE TABLE Import data from FILE into TABLE
do_test shell1-3.11.1 {
catchcmd "test.db" ".import"
} {1 {Usage: .import FILE TABLE}}
} {/1 .ERROR: missing FILE argument.*/}
do_test shell1-3.11.2 {
catchcmd "test.db" ".import FOO"
} {1 {Usage: .import FILE TABLE}}
#do_test shell1-3.11.2 {
# catchcmd "test.db" ".import FOO BAR"
#} {1 {Error: no such table: BAR}}
} {/1 .ERROR: missing TABLE argument.*/}
do_test shell1-3.11.3 {
# too many arguments
catchcmd "test.db" ".import FOO BAR BAD"
} {1 {Usage: .import FILE TABLE}}
} {/1 .ERROR: extra argument: "BAD".*./}
# .indexes ?TABLE? Show names of all indexes
# If TABLE specified, only show indexes for tables

9
test/shell5.test
View File

@ -32,17 +32,14 @@ forcedelete test.db test.db-journal test.db-wal
# .import FILE TABLE Import data from FILE into TABLE
do_test shell5-1.1.1 {
catchcmd "test.db" ".import"
} {1 {Usage: .import FILE TABLE}}
} {/1 .ERROR: missing FILE argument.*/}
do_test shell5-1.1.2 {
catchcmd "test.db" ".import FOO"
} {1 {Usage: .import FILE TABLE}}
#do_test shell5-1.1.2 {
# catchcmd "test.db" ".import FOO BAR"
#} {1 {Error: no such table: BAR}}
} {/1 .ERROR: missing TABLE argument.*/}
do_test shell5-1.1.3 {
# too many arguments
catchcmd "test.db" ".import FOO BAR BAD"
} {1 {Usage: .import FILE TABLE}}
} {/1 .ERROR: extra argument.*/}
# .separator STRING Change separator used by output mode and .import
do_test shell5-1.2.1 {