()
- {
- private boolean unread = true;
-
- @Override
- public void close() throws IOException
- {}
-
- @Override
- public NullWritable createKey()
- {
- return NullWritable.get();
- }
-
- @Override
- public NullWritable createValue()
- {
- return NullWritable.get();
- }
-
- @Override
- public long getPos() throws IOException
- {
- return 0;
- }
-
- @Override
- public float getProgress() throws IOException
- {
- return unread ? 0 : 1;
- }
-
- @Override
- /* spawn a cpimport process for each input file */
- public boolean next(NullWritable arg0, NullWritable arg1) throws IOException
- {
- InfiniDBConfiguration dbConf = new InfiniDBConfiguration(job);
- String schemaName = dbConf.getOutputSchemaName();
- String tableName = (filename.substring(filename.lastIndexOf('/')+1, filename.length()));
- tableName = tableName.substring(0, tableName.lastIndexOf('.'));
- String output = job.get("mapred.output.dir");
- if (unread)
- {
- try
- {
- StringBuilder loadCmdStr = new StringBuilder();
- loadCmdStr.append(dbConf.getInfiniDBHome());
- loadCmdStr.append("/bin/");
- loadCmdStr.append("infinidoop_load.sh ");
- loadCmdStr.append(filename);
- loadCmdStr.append(" ");
- loadCmdStr.append(schemaName);
- loadCmdStr.append(" ");
- loadCmdStr.append(tableName);
-
- Process lChldProc = Runtime.getRuntime().exec(loadCmdStr.toString());
-
- // Wait for the child to exit
- lChldProc.waitFor();
- BufferedReader lChldProcOutStream = new BufferedReader(new InputStreamReader(lChldProc.getInputStream()));
- BufferedReader stdError = new BufferedReader(new InputStreamReader(lChldProc.getErrorStream()));
-
- String lChldProcOutPutStr = null;
- StringBuffer outpath = new StringBuffer();
- outpath.append(job.getWorkingDirectory());
- outpath.append("/");
- outpath.append(output);
- outpath.append("/");
- outpath.append(tableName);
- outpath.append(".log");
-
- Path pt=new Path(outpath.toString());
- FileSystem fs = FileSystem.get(new Configuration());
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(pt, false)));
-
- // catch output
- while ((lChldProcOutPutStr = lChldProcOutStream.readLine()) != null)
- {
- br.write(lChldProcOutPutStr);
- br.newLine();
- }
-
- // catch error
- while ((lChldProcOutPutStr = stdError.readLine()) != null)
- {
- br.write(lChldProcOutPutStr);
- br.newLine();
- }
-
- //br.write(outpath.toString());
- //br.newLine();
- //br.write(loadCmdStr.toString());
- //br.newLine();
- //br.write(filename);
- br.close();
-
- lChldProcOutStream.close();
- }
- catch(Exception e)
- {
- e.printStackTrace();
- }
- unread = false;
- return true;
- }
- else
- {
- return false;
- }
- }
- };
- }
-
- @Override
- protected boolean isSplitable(FileSystem fs, Path filename)
- {
- return false;
- }
-
- }
diff --git a/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBConfiguration.java b/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBConfiguration.java
deleted file mode 100755
index 0c3d585bc..000000000
--- a/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBConfiguration.java
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
- * Copyright (c) 2014 InfiniDB, Inc.
- *
- * InfiniDB, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package infinidb.hadoop.db;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.lib.db.DBInputFormat.NullDBWritable;
-import org.apache.hadoop.mapred.lib.db.*;
-
-
-/**
- * A container for configuration property names for jobs with DB input/output.
- *
- * The job can be configured using the static methods in this class,
- * {@link DBInputFormat}, and {@link DBOutputFormat}.
- *
- * Alternatively, the properties can be set in the configuration with proper
- * values.
- *
- * @see DBConfiguration#configureDB(JobConf, String, String, String, String)
- * @see DBInputFormat#setInput(JobConf, Class, String, String)
- * @see DBInputFormat#setInput(JobConf, Class, String, String, String, String...)
- * @see DBOutputFormat#setOutput(JobConf, String, String...)
- */
-public class InfiniDBConfiguration{
-
-/** Input schema name */
-public static final String INPUT_SCHEMA_NAME_PROPERTY = "idb_hadoop.input.schema.name";
-
-/** Output schema name */
-public static final String OUTPUT_SCHEMA_NAME_PROPERTY = "idb_hadoop.output.schema.name";
-
-/** Output table name */
-public static final String OUTPUT_TABLE_NAMES_PROPERTY = "idb_hadoop.output.table.name";
-
-/** @InfiniDB Split key for split the query task */
-public static final String INPUT_SPLITKEY_NAME_PROPERTY = "idb_hadoop.splitkey.name";
-
-/** @InfiniDB Split key min value */
-public static final String INPUT_SPLITKEY_MIN_VAL = "idb_hadoop.splitkey.min.value";
-
-/** @InfiniDB Split key max value */
-public static final String INPUT_SPLITKEY_MAX_VAL = "idb_hadoop.splitkey.max.value";
-
-/** @InfiniDB HOME path */
-public static final String INFINIDB_HOME = "idb_hadoop.infinidb.home.path";
-
-/** Input dir */
-public static final String INPUT_PATH = "mapred.input.dir";
-
-/** Output dir */
-public static final String OUTPUT_PATH = "mapred.output.dir";
-
-/**
- * Sets the DB access related fields in the JobConf.
- * @param job the job
- * @param driverClass JDBC Driver class name
- * @param dbUrl JDBC DB access URL.
- * @param userName DB access username
- * @param passwd DB access passwd
- */
-public static void configureDB(JobConf job, String driverClass, String dbUrl
- , String userName, String passwd)
-{
-
- job.set(DBConfiguration.DRIVER_CLASS_PROPERTY, driverClass);
- job.set(DBConfiguration.URL_PROPERTY, dbUrl);
- if(userName != null)
- job.set(DBConfiguration.USERNAME_PROPERTY, userName);
- if(passwd != null)
- job.set(DBConfiguration.PASSWORD_PROPERTY, passwd);
-}
-
-/**
- * Sets the DB access related fields in the JobConf.
- * @param job the job
- * @param driverClass JDBC Driver class name
- * @param dbUrl JDBC DB access URL.
- */
-public static void configureDB(JobConf job, String driverClass, String dbUrl)
-{
- configureDB(job, driverClass, dbUrl, null, null);
-}
-
-private JobConf job;
-
-public InfiniDBConfiguration(JobConf job)
-{
- this.job = job;
-}
-
-/** Returns a connection object o the DB
- * @throws ClassNotFoundException
- * @throws SQLException
- */
-Connection getConnection() throws IOException
-{
- try
- {
- Class.forName(job.get(DBConfiguration.DRIVER_CLASS_PROPERTY));
- }catch (ClassNotFoundException exception)
- {
- throw new IOException("Conection driver can not be loaded", exception);
- }
-
- try
- {
- if(job.get(DBConfiguration.USERNAME_PROPERTY) == null)
- {
- return DriverManager.getConnection(job.get(DBConfiguration.URL_PROPERTY));
- }
- else
- {
- return DriverManager.getConnection(
- job.get(DBConfiguration.URL_PROPERTY),
- job.get(DBConfiguration.USERNAME_PROPERTY),
- job.get(DBConfiguration.PASSWORD_PROPERTY));
- }
- }catch (SQLException exception)
- {
- throw new IOException("Conection can not be established", exception);
- }
-}
-
-String getInputSchemaName()
-{
- return job.get(InfiniDBConfiguration.INPUT_SCHEMA_NAME_PROPERTY);
-}
-
-void setInputSchemaName(String schemaName)
-{
- job.set(InfiniDBConfiguration.INPUT_SCHEMA_NAME_PROPERTY, schemaName);
-}
-
-String getInputTableName()
-{
- return job.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY);
-}
-
-void setInputTableName(String tableName)
-{
- job.set(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, tableName);
-}
-
-String[] getInputFieldNames()
-{
- return job.getStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY);
-}
-
-void setInputFieldNames(String... fieldNames)
-{
- job.setStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, fieldNames);
-}
-
-String getInputConditions()
-{
- return job.get(DBConfiguration.INPUT_CONDITIONS_PROPERTY);
-}
-
-void setInputConditions(String conditions)
-{
- if (conditions != null && conditions.length() > 0)
- job.set(DBConfiguration.INPUT_CONDITIONS_PROPERTY, conditions);
-}
-
-String getInputOrderBy()
-{
- return job.get(DBConfiguration.INPUT_ORDER_BY_PROPERTY);
-}
-
-/** @InfiniDB */
-void setSplitKey(String key)
-{
- job.setStrings(InfiniDBConfiguration.INPUT_SPLITKEY_NAME_PROPERTY, key);
-}
-
-/** @InfiniDB */
-String getSplitKey()
-{
- return job.get(InfiniDBConfiguration.INPUT_SPLITKEY_NAME_PROPERTY);
-}
-
-/** @InfiniDB */
-public void setMinVal(long value)
-{
- job.setLong(INPUT_SPLITKEY_MIN_VAL, value);
-}
-
-/** @InfiniDB */
-public Long getMinVal()
-{
- if(job.get(INPUT_SPLITKEY_MIN_VAL)==null)
- return null;
- return job.getLong(INPUT_SPLITKEY_MIN_VAL, -1);
-}
-
-/** @InfiniDB */
-public void setMaxVal(long value)
-{
- job.setFloat(INPUT_SPLITKEY_MAX_VAL, value);
-}
-
-/** @InfiniDB */
-public Long getMaxVal()
-{
- if(job.get(INPUT_SPLITKEY_MAX_VAL)==null)
- return null;
- return job.getLong(INPUT_SPLITKEY_MAX_VAL, -1);
-}
-
-void setInputOrderBy(String orderby)
-{
- if(orderby != null && orderby.length() >0)
- {
- job.set(DBConfiguration.INPUT_ORDER_BY_PROPERTY, orderby);
- }
-}
-
-String getInputQuery()
-{
- return job.get(DBConfiguration.INPUT_QUERY);
-}
-
-void setInputQuery(String query)
-{
- if(query != null && query.length() >0)
- {
- job.set(DBConfiguration.INPUT_QUERY, query);
- }
-}
-
-String getInputCountQuery()
-{
- return job.get(DBConfiguration.INPUT_COUNT_QUERY);
-}
-
-void setInputCountQuery(String query)
-{
- if(query != null && query.length() >0)
- {
- job.set(DBConfiguration.INPUT_COUNT_QUERY, query);
- }
-}
-
-
-Class> getInputClass()
-{
- return job.getClass(DBConfiguration.INPUT_CLASS_PROPERTY, NullDBWritable.class);
-}
-
-void setInputClass(Class extends DBWritable> inputClass)
-{
- job.setClass(DBConfiguration.INPUT_CLASS_PROPERTY, inputClass, DBWritable.class);
-}
-
-String getOutputSchemaName()
-{
- return job.get(InfiniDBConfiguration.OUTPUT_SCHEMA_NAME_PROPERTY);
-}
-
-void setOutputSchemaName(String schemaName)
-{
- job.set(InfiniDBConfiguration.OUTPUT_SCHEMA_NAME_PROPERTY, schemaName);
-}
-
-String[] getOutputTableNames()
-{
- return job.getStrings(InfiniDBConfiguration.OUTPUT_TABLE_NAMES_PROPERTY);
-}
-
-void setOutputTableNames(String... tableNames)
-{
- job.setStrings(InfiniDBConfiguration.OUTPUT_TABLE_NAMES_PROPERTY, tableNames);
-}
-
-String[] getOutputFieldNames()
-{
- return job.getStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY);
-}
-
-void setOutputFieldNames(String... fieldNames)
-{
- job.setStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, fieldNames);
-}
-
-public String getInfiniDBHome()
-{
- return job.get(InfiniDBConfiguration.INFINIDB_HOME);
-}
-
-public void setInfiniDBHome(String path)
-{
- job.set(InfiniDBConfiguration.INFINIDB_HOME, path);
-}
-
-public String getInputPath()
-{
- return job.get(InfiniDBConfiguration.INPUT_PATH);
-}
-
-public void setInputPath(String path)
-{
- job.set(InfiniDBConfiguration.INPUT_PATH, path);
-}
-
-public String getOutputPath()
-{
- return job.get(InfiniDBConfiguration.OUTPUT_PATH);
-}
-
-public void setOutputPath(String path)
-{
- job.set(InfiniDBConfiguration.OUTPUT_PATH, path);
-}
-
-}
-
-
diff --git a/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBInputFormat.java b/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBInputFormat.java
deleted file mode 100755
index 5909a80fd..000000000
--- a/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBInputFormat.java
+++ /dev/null
@@ -1,442 +0,0 @@
-/*
- * Copyright (c) 2014 InfiniDB, Inc.
- *
- * InfiniDB, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package infinidb.hadoop.db;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobConfigurable;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.mapred.lib.db.*;
-
-
-/**
- * A InputFormat that reads input data from an SQL table.
- *
- * DBInputFormat emits LongWritables containing the record number as
- * key and DBWritables as value.
- *
- * The SQL query, and input class can be using one of the two
- * setInput methods.
- */
-public class InfiniDBInputFormat
- implements InputFormat, JobConfigurable
-{
- /**
- * A RecordReader that reads records from a SQL table.
- * Emits LongWritables containing the record number as
- * key and DBWritables as value.
- */
-protected class DBRecordReader implements RecordReader
-{
- private ResultSet results;
-
- private Statement statement;
-
- private Class inputClass;
-
- private JobConf job;
-
- private InfiniDBInputSplit split;
-
- private long pos = 0;
-
- /**
- * @param split The InputSplit to read data for
- * @throws SQLException
- */
- protected DBRecordReader(InfiniDBInputSplit split, Class inputClass, JobConf job) throws SQLException
- {
- this.inputClass = inputClass;
- this.split = split;
- this.job = job;
-
- statement = connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
-
- //statement.setFetchSize(Integer.MIN_VALUE);
- results = statement.executeQuery(getSelectQuery());
- }
-
- /** @InfiniDB */
- public String concat(S[] arr, String sep)
- {
- String ret = "";
- for(int i=0; i < arr.length; i++)
- {
- ret = ret + arr[i];
- if(i < arr.length-1)
- {
- ret = ret + sep;
- }
- }
- return ret;
- }
-
- /** @InfiniDB Returns the query for selecting the records,
- * subclasses can override this for custom behaviour.*/
- protected String getSelectQuery()
- {
- InfiniDBConfiguration conf = new InfiniDBConfiguration(job);
- StringBuilder query = new StringBuilder();
- query.append("SELECT ");
- query.append(concat(conf.getInputFieldNames(), ","));
- query.append(" FROM ");
- query.append(conf.getInputTableName());
- query.append(" WHERE ");
- query.append(split.splitKey + ">=" + split.getStart());
- query.append(" AND ");
- query.append(split.splitKey + "<" + split.getEnd());
- if (conditions != null && conditions.length() > 0)
- query.append(" AND (").append(conditions).append(")");
- return query.toString();
- }
-
-/** {@inheritDoc} */
-public void close() throws IOException
-{
- try
- {
- connection.commit();
- results.close();
- statement.close();
- } catch (SQLException e)
- {
- throw new IOException(e.getMessage());
- }
-}
-
- /** {@inheritDoc} */
-public LongWritable createKey()
-{
- return new LongWritable();
-}
-
-/** {@inheritDoc} */
-public T createValue()
-{
- return ReflectionUtils.newInstance(inputClass, job);
-}
-
-/** {@inheritDoc} */
-public long getPos() throws IOException
-{
- return pos;
-}
-
-/** {@inheritDoc} */
-public float getProgress() throws IOException
-{
- return pos / (float)split.getLength();
-}
-
-/** {@inheritDoc} */
-public boolean next(LongWritable key, T value) throws IOException
-{
- try
- {
- if (!results.next())
- return false;
-
- // Set the key field value as the output key value
- key.set(pos + split.getStart());
-
- value.readFields(results);
-
- pos ++;
- } catch (SQLException e)
- {
- throw new IOException(e.getMessage());
- }
- return true;
-}
-}
-
- /**
- * A Class that does nothing, implementing DBWritable
- */
- public static class NullDBWritable implements DBWritable, Writable {
- @Override
- public void readFields(DataInput in) throws IOException { }
- @Override
- public void readFields(ResultSet arg0) throws SQLException { }
- @Override
- public void write(DataOutput out) throws IOException { }
- @Override
- public void write(PreparedStatement arg0) throws SQLException { }
- }
- /**
- * A InputSplit that spans a set of rows
- */
- protected static class InfiniDBInputSplit implements InputSplit {
-
- private long end = 0;
- private long start = 0;
- private String splitKey;
-
- /**
- * Default Constructor
- */
- public InfiniDBInputSplit() {
- }
-
- /**
- * @InfiniDB
- * Convenience Constructor
- * @param start the index of the first row to select
- * @param end the index of the last row to select
- */
- public InfiniDBInputSplit(long start, long end, String key) {
- this.start = start;
- this.end = end;
- this.splitKey = key;
- }
-
- /** {@inheritDoc} */
- public String[] getLocations() throws IOException {
- return new String[] {};
- }
-
- /**
- * @return The index of the first row to select
- */
- public long getStart() {
- return start;
- }
-
- /**
- * @return The index of the last row to select
- */
- public long getEnd() {
- return end;
- }
-
- /**
- * @return The total row count in this split
- */
- public long getLength() throws IOException {
- return end - start;
- }
-
- /** {@inheritDoc} */
- public void readFields(DataInput input) throws IOException {
- start = input.readLong();
- end = input.readLong();
- splitKey = WritableUtils.readString(input);
- }
-
- /** {@inheritDoc} */
- public void write(DataOutput output) throws IOException {
- output.writeLong(start);
- output.writeLong(end);
- WritableUtils.writeString(output, splitKey);
- }
- }
-
- private String conditions;
-
- private Connection connection;
-
- private String tableName;
-
- private String[] fieldNames;
-
- private InfiniDBConfiguration dbConf;
-
- /** {@inheritDoc} */
- public void configure(JobConf job) {
-
- dbConf = new InfiniDBConfiguration(job);
-
- try {
- this.connection = dbConf.getConnection();
- this.connection.setAutoCommit(false);
- connection.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
- }
- catch (Exception ex) {
- throw new RuntimeException(ex);
- }
-
- tableName = dbConf.getInputTableName();
- fieldNames = dbConf.getInputFieldNames();
- conditions = dbConf.getInputConditions();
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- public RecordReader getRecordReader(InputSplit split,
- JobConf job, Reporter reporter) throws IOException {
-
- Class inputClass = dbConf.getInputClass();
- try {
- return new DBRecordReader((InfiniDBInputSplit) split, inputClass, job);
- }
- catch (SQLException ex) {
- throw new IOException(ex.getMessage());
- }
- }
-
- /** @InfiniDB */
- private long getMaxVal(InfiniDBConfiguration conf, Connection conn, String tableName, String col) {
- if(conf.getMaxVal()!=null) {
- return conf.getMaxVal();
- }
- try {
- PreparedStatement s = conn.prepareStatement("SELECT MAX(" + col + ") FROM " + tableName);
- ResultSet rs = s.executeQuery();
- rs.next();
- long ret = rs.getLong(1);
- rs.close();
- s.close();
- return ret;
- } catch(SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
- /** @InfiniDB */
- private long getMinVal(InfiniDBConfiguration conf, Connection conn, String tableName, String col ) {
- if(conf.getMinVal()!=null) {
- return conf.getMinVal();
- }
- try {
- PreparedStatement s = conn.prepareStatement("SELECT MIN(" + col + ") FROM " + tableName);
- ResultSet rs = s.executeQuery();
- rs.next();
- long ret = rs.getLong(1);
- rs.close();
- s.close();
- return ret;
- } catch(SQLException e) {
- throw new RuntimeException(e);
- }
- }
-
-
- /** {@inheritDoc}
- * @InfiniDB
- */
- public InputSplit[] getSplits(JobConf job, int chunks) throws IOException {
-
- try {
- InfiniDBConfiguration conf = new InfiniDBConfiguration(job);
- Connection conn = conf.getConnection();
- String splitKey = conf.getSplitKey();
- long maxVal = getMaxVal(conf, conn, conf.getInputTableName(), conf.getSplitKey());
- long minVal = getMinVal(conf, conn, conf.getInputTableName(), conf.getSplitKey());
- System.out.println("max=" + maxVal);
- System.out.println("min=" + minVal);
-
- InputSplit[] ret = new InputSplit[chunks];
- long chunkSize = (maxVal - minVal + 1) / chunks + 1;
- long start = minVal;
- for (int i = 0; i < chunks; i++){
- ret[i] = new InfiniDBInputSplit(start, start+chunkSize, splitKey);
- start += chunkSize;
- }
-
- conn.close();
- return ret;
- } catch(SQLException e) {
- throw new RuntimeException(e);
- }
-
-}
-
- /** Returns the query for getting the total number of rows,
- * subclasses can override this for custom behaviour.*/
- protected String getCountQuery() {
-
- if(dbConf.getInputCountQuery() != null) {
- return dbConf.getInputCountQuery();
- }
-
- StringBuilder query = new StringBuilder();
- query.append("SELECT COUNT(*) FROM " + tableName);
-
- if (conditions != null && conditions.length() > 0)
- query.append(" WHERE " + conditions);
- return query.toString();
- }
-
- /**
- * @InfiniDB
- * Initializes the map-part of the job with the appropriate input settings.
- *
- * @param job The job
- * @param inputClass the class object implementing DBWritable, which is the
- * Java object holding tuple fields.
- * @param tableName The table to read data from
- * @param conditions The condition which to select data with, eg. '(updated >
- * 20070101 AND length > 0)'
- * @param key the field name used for split key.
- * @param fieldNames The field names in the table
- * @see #setInput(JobConf, Class, String, String)
- */
- public static void setInput(JobConf job, Class extends DBWritable> inputClass,
- String tableName,String conditions, String key, String... fieldNames) {
-
- job.setInputFormat(InfiniDBInputFormat.class);
- InfiniDBConfiguration dbConf = new InfiniDBConfiguration(job);
- dbConf.setInputClass(inputClass);
- dbConf.setInputTableName(tableName);
- dbConf.setInputFieldNames(fieldNames);
- dbConf.setInputConditions(conditions);
- dbConf.setSplitKey(key);
- }
-
- /**
- * @InfiniDB
- * Initializes the map-part of the job with the appropriate input settings.
- *
- * @param job The job
- * @param inputClass the class object implementing DBWritable, which is the
- * Java object holding tuple fields.
- * @param inputQuery the input query to select fields. Example :
- * "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
- * @param inputCountQuery the input query that returns the number of records in
- * the table.
- * Example : "SELECT COUNT(f1) FROM Mytable"
- * @see #setInput(JobConf, Class, String, String, String, String...)
- */
- public static void setInput(JobConf job, Class extends DBWritable> inputClass,
- String inputQuery, String inputCountQuery) {
- job.setInputFormat(InfiniDBInputFormat.class);
-
- InfiniDBConfiguration dbConf = new InfiniDBConfiguration(job);
- dbConf.setInputClass(inputClass);
- dbConf.setInputQuery(inputQuery);
- dbConf.setInputCountQuery(inputCountQuery);
-
- }
-
-}
diff --git a/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBOutputFormat.java b/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBOutputFormat.java
deleted file mode 100755
index 2427df45c..000000000
--- a/utils/infinidb_hadoop/src/infinidb/hadoop/db/InfiniDBOutputFormat.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Copyright (c) 2014 InfiniDB, Inc.
- *
- * InfiniDB, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package infinidb.hadoop.db;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hadoop.mapred.RecordWriter;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.mapred.lib.db.*;
-
-
-/**
- * A OutputFormat that sends the reduce output to a SQL table.
- *
- * {@link DBOutputFormat} accepts <key,value> pairs, where
- * key has a type extending DBWritable. Returned {@link RecordWriter}
- * writes only the key to the database with a batch SQL query.
- *
- */
-public class InfiniDBOutputFormat
-implements OutputFormat {
-
- private static final Log LOG = LogFactory.getLog(InfiniDBOutputFormat.class);
-
- /**
- * A RecordWriter that writes the reduce output to a SQL table
- */
- protected class DBRecordWriter
- implements RecordWriter {
-
- private Connection connection;
- private PreparedStatement statement;
-
- protected DBRecordWriter() throws SQLException
- {}
-
- /** {@inheritDoc} */
- public void close(Reporter reporter) throws IOException
- {}
-
- /** {@inheritDoc} */
- public void write(K key, V value) throws IOException
- {}
- }
-
- /** {@inheritDoc} */
- public void checkOutputSpecs(FileSystem filesystem, JobConf job)
- throws IOException
- {}
-
-
-/** {@inheritDoc} */
-public RecordWriter getRecordWriter(FileSystem filesystem,
- JobConf job, String name, Progressable progress) throws IOException
-{
- try {
- return new DBRecordWriter();
- }
- catch (Exception ex) {
- throw new IOException(ex.getMessage());
- }
-}
-
-/**
- * Initializes the reduce-part of the job with the appropriate output settings
- *
- * @param job
- * The job
- * @param tableName
- * The table to insert data into
- * @param fieldNames
- * The field names in the table. If unknown, supply the appropriate
- * number of nulls.
- */
-public static void setOutput(JobConf job, String schemaName, String ... tableNames)
-{
- job.setOutputFormat(InfiniDBOutputFormat.class);
- job.setReduceSpeculativeExecution(false);
-
- InfiniDBConfiguration dbConf = new InfiniDBConfiguration(job);
- dbConf.setOutputSchemaName(schemaName);
- dbConf.setOutputTableNames(tableNames);
-}
-
-/**
- * Initializes the reduce-part of the job with the appropriate output settings
- *
- * @param job
- * The job
- * @param tableName
- * The table to insert data into
- * @param fieldNames
- * The field names in the table. If unknown, supply the appropriate
- * number of nulls.
- */
-public static void setOutput(JobConf job, String schemaName)
-{
- job.setOutputFormat(InfiniDBOutputFormat.class);
- job.setReduceSpeculativeExecution(false);
-
- InfiniDBConfiguration dbConf = new InfiniDBConfiguration(job);
-
- dbConf.setOutputSchemaName(schemaName);
-}
-}
-
diff --git a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDBOutputDriver.java b/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDBOutputDriver.java
deleted file mode 100755
index 307624995..000000000
--- a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDBOutputDriver.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright (c) 2014 InfiniDB, Inc.
- *
- * InfiniDB, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package infinidb.hadoop.example;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.sql.*;
-import java.util.Date;
-import java.util.Formatter;
-import java.io.IOException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.mapred.lib.db.*;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.mapred.TextInputFormat;
-import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.*;
-import org.apache.hadoop.*;
-
-import infinidb.hadoop.db.*;
-
-public class InfiniDBOutputDriver extends Configured implements Tool
-{
- public int run (String[] args) throws Exception
- {
- Configuration conf = new Configuration();
- JobConf jobconf = new JobConf(conf, InfiniDoopDriver.class);
- DBConfiguration.configureDB(jobconf,
- "com.mysql.jdbc.Driver",
- "jdbc:mysql://srvswint4/tpch1","root", "");
- String [] fields = { "n_nationkey", "n_name" };
- String [] outFields = {"id", "name"};
- jobconf.setInputFormat(IDBFileInputFormat.class);
- jobconf.setOutputFormat(InfiniDBOutputFormat.class);
- jobconf.setOutputKeyClass(NullWritable.class);
- jobconf.setOutputValueClass(Text.class);
- InfiniDBOutputFormat.setOutput(jobconf, "db", outFields);
- InfiniDBConfiguration idbconf = new InfiniDBConfiguration(jobconf);
- idbconf.setInputPath("input");
- idbconf.setOutputPath("output");
- idbconf.setInfiniDBHome("/usr/local/mariadb/columnstore");
-
- jobconf.setMapperClass(InfiniDoopMapper.class);
- jobconf.setNumMapTasks(1);
- jobconf.setNumReduceTasks(2);
- JobClient client = new JobClient();
- client.setConf(jobconf);
- try {
- JobClient.runJob(jobconf);
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- return 0;
-}
-
-public static void main(String [] args) throws Exception
-{
- int ret = ToolRunner.run(new InfiniDBOutputDriver(), args);
- System.exit(ret);
-}
-
-}
diff --git a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopDriver.java b/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopDriver.java
deleted file mode 100755
index 74bce0ea3..000000000
--- a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopDriver.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Copyright (c) 2014 InfiniDB, Inc.
- *
- * InfiniDB, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package infinidb.hadoop.example;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.sql.*;
-import java.util.Date;
-import java.util.Formatter;
-import java.io.IOException;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.lib.db.*;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.input.*;
-import org.apache.hadoop.mapreduce.lib.output.*;
-import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.Reducer;
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.*;
-import org.apache.hadoop.*;
-
-import infinidb.hadoop.db.*;
-import infinidb.hadoop.db.InfiniDBConfiguration;
-
-public class InfiniDoopDriver extends Configured implements Tool
-{
- public int run (String[] args) throws Exception
- {
- Configuration conf = new Configuration();
- JobConf jobconf = new JobConf(conf, InfiniDoopDriver.class);
- DBConfiguration.configureDB(jobconf,
- "com.mysql.jdbc.Driver",
- "jdbc:mysql://srvswint4/tpch1","root", "");
- String [] fields = { "n_nationkey", "n_name" };
- jobconf.setInputFormat(InfiniDBInputFormat.class);
-
- jobconf.setOutputKeyClass(LongWritable.class);
- jobconf.setOutputValueClass(Text.class);
-
- InfiniDBInputFormat.setInput(jobconf, InfiniDoopRecord.class, "nation",
- null, "n_nationkey", fields);
-
- InfiniDBConfiguration idbconf = new InfiniDBConfiguration(jobconf);
- idbconf.setOutputPath("output2");
- jobconf.setMapperClass(InfiniDoopInputMapper.class);
- jobconf.setNumMapTasks(4);
- jobconf.setNumReduceTasks(1);
- jobconf.set("mapred.textoutputformat.separator", "|");
- JobClient client = new JobClient();
-
- client.setConf(jobconf);
- try {
- JobClient.runJob(jobconf);
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- return 0;
-}
- public static void main(String [] args) throws Exception
- {
- int ret = ToolRunner.run(new InfiniDoopDriver(), args);
- System.exit(ret);
- }
-
-}
diff --git a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopInputMapper.java b/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopInputMapper.java
deleted file mode 100755
index ebc8b6e30..000000000
--- a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopInputMapper.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2014 InfiniDB, Inc.
- *
- * InfiniDB, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package infinidb.hadoop.example;
-
-import java.io.IOException;
-import java.io.*;
-import java.sql.*;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.lib.db.*;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
-public class InfiniDoopInputMapper extends MapReduceBase implements
- Mapper {
-
- public void map(LongWritable key, InfiniDoopRecord val,
- OutputCollector output, Reporter reporter) throws IOException {
- output.collect(new LongWritable(val.id), new Text(val.name));
- }
-
-}
diff --git a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopMapper.java b/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopMapper.java
deleted file mode 100755
index 5efd50631..000000000
--- a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopMapper.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright (c) 2014 InfiniDB, Inc.
- *
- * InfiniDB, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package infinidb.hadoop.example;
-
-import java.io.IOException;
-import java.io.*;
-import java.sql.*;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.lib.db.*;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.*;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
-/** Dummy mapper, basically doing nothing. the real job is invoked by input format */
-public class InfiniDoopMapper extends MapReduceBase implements
- Mapper {
-
- public void map(NullWritable key, NullWritable val,
- OutputCollector output, Reporter reporter) throws IOException {
- NullWritable n = NullWritable.get();
- output.collect(n, n);
- }
-}
-
diff --git a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopRecord.java b/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopRecord.java
deleted file mode 100755
index 2bc370ff1..000000000
--- a/utils/infinidb_hadoop/src/infinidb/hadoop/example/InfiniDoopRecord.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2014 InfiniDB, Inc.
- *
- * InfiniDB, Inc. licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package infinidb.hadoop.example;
-
-import java.io.IOException;
-import java.io.*;
-import java.sql.*;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.lib.db.*;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.MapReduceBase;
-import org.apache.hadoop.mapred.Mapper;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-
-public class InfiniDoopRecord implements Writable, DBWritable, WritableComparable {
- long id;
- String name;
-
- public void readFields(DataInput in) throws IOException {
- this.id = in.readLong();
- this.name = Text.readString(in);
- }
-
- public void readFields(ResultSet resultSet)
- throws SQLException {
- this.id = resultSet.getLong(1);
- this.name = resultSet.getString(2);
- }
-
- public void write(DataOutput out) throws IOException {
- out.writeLong(this.id);
- Text.writeString(out, this.name);
- }
-
- public void write(PreparedStatement stmt) throws SQLException {
- stmt.setLong(1, this.id);
- stmt.setString(2, this.name);
- }
-
- public int compareTo(InfiniDoopRecord w) {
- return (this.id < w.id ? -1 :(this.id == w.id ? 0 : 1));
- }
- }
diff --git a/utils/loggingcpp/CMakeLists.txt b/utils/loggingcpp/CMakeLists.txt
index be2a058f9..b44cd8b21 100644
--- a/utils/loggingcpp/CMakeLists.txt
+++ b/utils/loggingcpp/CMakeLists.txt
@@ -30,5 +30,5 @@ set_target_properties(loggingcpp PROPERTIES VERSION 1.0.0 SOVERSION 1)
install(TARGETS loggingcpp DESTINATION ${ENGINE_LIBDIR} COMPONENT libs)
-install(FILES MessageFile.txt ErrorMessage.txt DESTINATION ${ENGINE_ETCDIR} COMPONENT platform)
+install(FILES MessageFile.txt ErrorMessage.txt DESTINATION ${ENGINE_SYSCONFDIR}/columnstore COMPONENT platform)
diff --git a/utils/loggingcpp/idberrorinfo.cpp b/utils/loggingcpp/idberrorinfo.cpp
index 4932634c0..a2e9056d2 100644
--- a/utils/loggingcpp/idberrorinfo.cpp
+++ b/utils/loggingcpp/idberrorinfo.cpp
@@ -34,6 +34,7 @@ using namespace std;
#include
using namespace boost;
+#include "config.h"
#include "configcpp.h"
using namespace config;
#include "loggingid.h"
@@ -64,7 +65,7 @@ IDBErrorInfo::IDBErrorInfo()
string configFile(cf->getConfig("SystemConfig", "ErrorMessageFile"));
if (configFile.length() == 0)
- configFile = startup::StartUp::installDir() + "/etc/ErrorMessage.txt";
+ configFile = std::string(MCSSYSCONFDIR) + "/columnstore/ErrorMessage.txt";
ifstream msgFile(configFile.c_str());
diff --git a/utils/loggingcpp/message.cpp b/utils/loggingcpp/message.cpp
index 030567e4b..7f1468594 100644
--- a/utils/loggingcpp/message.cpp
+++ b/utils/loggingcpp/message.cpp
@@ -34,6 +34,7 @@ using namespace std;
#include
using namespace boost;
+#include "config.h"
#include "configcpp.h"
using namespace config;
#include "messageobj.h"
@@ -56,7 +57,7 @@ void loadCatalog()
string configFile(cf->getConfig("MessageLog", "MessageLogFile"));
if (configFile.length() == 0)
- configFile = startup::StartUp::installDir() + "/etc/MessageFile.txt";
+ configFile = std::string(MCSSYSCONFDIR) + "/columnstore/MessageFile.txt";
ifstream msgFile(configFile.c_str());
diff --git a/utils/multicast/config.h b/utils/multicast/config.h
deleted file mode 100644
index 1596ab1d9..000000000
--- a/utils/multicast/config.h
+++ /dev/null
@@ -1,208 +0,0 @@
-/* Copyright (C) 2014 InfiniDB, Inc.
-
- This program is free software; you can redistribute it and/or
- modify it under the terms of the GNU General Public License
- as published by the Free Software Foundation; version 2 of
- the License.
-
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
- MA 02110-1301, USA. */
-
-/* config.h. Generated from config.h.in by configure. */
-/* config.h.in. Generated from configure.in by autoheader. */
-
-/* Define to 1 if you have the header file. */
-#define HAVE_ARPA_INET_H 1
-
-/* Define to 1 if you have the `atexit' function. */
-#define HAVE_ATEXIT 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_DLFCN_H 1
-
-/* Define to 1 if you have the `dlsym' function. */
-#define HAVE_DLSYM 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_FCNTL_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_GETOPT_H 1
-
-/* Define to 1 if you have the `getopt_long' function. */
-#define HAVE_GETOPT_LONG 1
-
-/* Define to 1 if you have the `htons' function. */
-#define HAVE_HTONS 1
-
-/* Define to 1 if you have the `inet_aton' function. */
-#define HAVE_INET_ATON 1
-
-/* Define to 1 if you have the `inet_pton' function. */
-#define HAVE_INET_PTON 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_INTTYPES_H 1
-
-/* Define to 1 if the system has the type `in_addr_t'. */
-#define HAVE_IN_ADDR_T 1
-
-/* Define to 1 if you have the `dl' library (-ldl). */
-#define HAVE_LIBDL 1
-
-/* Define to 1 if you have the `pthread' library (-lpthread). */
-#define HAVE_LIBPTHREAD 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_LIMITS_H 1
-
-/* Define when you have an LLSEEK prototype */
-/* #undef HAVE_LLSEEK_PROTOTYPE */
-
-/* Define when the compiler supports LOFF_T type */
-#define HAVE_LOFF_T 1
-
-/* Define when the compiler supports LONG_LONG type */
-#define HAVE_LONG_LONG 1
-
-/* Define to 1 if you have the `lseek64' function. */
-#define HAVE_LSEEK64 1
-
-/* Define when you have an LSEEK64 prototype */
-#define HAVE_LSEEK64_PROTOTYPE 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_MALLOC_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_MEMORY_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_NETDB_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_NETINET_IN_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_NET_IF_H 1
-
-/* Define when the compiler supports OFFSET_T type */
-/* #undef HAVE_OFFSET_T */
-
-/* Define when the system has a 64 bit off_t type */
-#define HAVE_OFF_T_64 1
-
-/* Define to 1 if you have the `on_exit' function. */
-#define HAVE_ON_EXIT 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SIGNAL_H 1
-
-/* Define to 1 if you have the `snprintf' function. */
-#define HAVE_SNPRINTF 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STDINT_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STDLIB_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STRINGS_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STRING_H 1
-
-/* Define to 1 if `imr_ifindex' is member of `struct ip_mreqn'. */
-#define HAVE_STRUCT_IP_MREQN_IMR_IFINDEX 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_IOCTL_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_PARAM_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_SELECT_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_SOCKET_H 1
-
-/* Define to 1 if you have the header file. */
-/* #undef HAVE_SYS_SOCKIO_H */
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_STAT_H 1
-
-/* Define to 1 if you have the header file. */
-/* #undef HAVE_SYS_TERMIOS_H */
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_TIME_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_TYPES_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_UIO_H 1
-
-/* Define to 1 if you have that is POSIX.1 compatible. */
-#define HAVE_SYS_WAIT_H 1
-
-/* Define to 1 if you have the `tcsetattr' function. */
-#define HAVE_TCSETATTR 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_TERMIOS_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_UNISTD_H 1
-
-/* Define to 1 if you have the header file. */
-/* #undef HAVE_WINSOCK2_H */
-
-/* Define to the address where bug reports for this package should be sent. */
-#define PACKAGE_BUGREPORT ""
-
-/* Define to the full name of this package. */
-#define PACKAGE_NAME ""
-
-/* Define to the full name and version of this package. */
-#define PACKAGE_STRING ""
-
-/* Define to the one symbol short name of this package. */
-#define PACKAGE_TARNAME ""
-
-/* Define to the version of this package. */
-#define PACKAGE_VERSION ""
-
-/* Define as the return type of signal handlers (`int' or `void'). */
-#define RETSIGTYPE void
-
-/* Define to 1 if you have the ANSI C header files. */
-#define STDC_HEADERS 1
-
-/* Define to 1 if you can safely include both and . */
-#define TIME_WITH_SYS_TIME 1
-
-/* Define to 1 if your declares `struct tm'. */
-/* #undef TM_IN_SYS_TIME */
-
-/* Define to empty if `const' does not conform to ANSI C. */
-/* #undef const */
-
-/* Define to `__inline__' or `__inline' if that's what the C compiler
- calls it, or to nothing if 'inline' is not supported under any name. */
-#ifndef __cplusplus
-/* #undef inline */
-#endif
-
-/* Define to `unsigned int' if does not define. */
-/* #undef size_t */
diff --git a/utils/thrift/thrift/config.h b/utils/thrift/thrift/config.h
deleted file mode 100644
index e60e4c1ca..000000000
--- a/utils/thrift/thrift/config.h
+++ /dev/null
@@ -1,427 +0,0 @@
-/* lib/cpp/src/thrift/config.h. Generated from config.hin by configure. */
-/* config.hin. Generated from configure.ac by autoheader. */
-
-
-#ifndef CONFIG_H
-#define CONFIG_H
-
-
-/* Define if the AI_ADDRCONFIG symbol is unavailable */
-/* #undef AI_ADDRCONFIG */
-
-/* Possible value for SIGNED_RIGHT_SHIFT_IS */
-#define ARITHMETIC_RIGHT_SHIFT 1
-
-/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP
- systems. This function is required for `alloca.c' support on those systems.
- */
-/* #undef CRAY_STACKSEG_END */
-
-/* Define to 1 if using `alloca.c'. */
-/* #undef C_ALLOCA */
-
-/* Define to 1 if you have the `alarm' function. */
-#define HAVE_ALARM 1
-
-/* Define to 1 if you have `alloca', as a function or macro. */
-#define HAVE_ALLOCA 1
-
-/* Define to 1 if you have and it should be used (not on Ultrix).
- */
-#define HAVE_ALLOCA_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_ARPA_INET_H 1
-
-/* define if the Boost library is available */
-#define HAVE_BOOST /**/
-
-/* Define to 1 if you have the `bzero' function. */
-#define HAVE_BZERO 1
-
-/* Define to 1 if you have the `clock_gettime' function. */
-#define HAVE_CLOCK_GETTIME 1
-
-/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you
- don't. */
-#define HAVE_DECL_STRERROR_R 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_DLFCN_H 1
-
-/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */
-/* #undef HAVE_DOPRNT */
-
-/* Define to 1 if you have the header file. */
-#define HAVE_FCNTL_H 1
-
-/* Define to 1 if you have the `fork' function. */
-#define HAVE_FORK 1
-
-/* Define to 1 if you have the `ftruncate' function. */
-#define HAVE_FTRUNCATE 1
-
-/* Define to 1 if you have the `gethostbyname' function. */
-#define HAVE_GETHOSTBYNAME 1
-
-/* Define to 1 if you have the `gettimeofday' function. */
-#define HAVE_GETTIMEOFDAY 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_INTTYPES_H 1
-
-/* define if libevent is available */
-/* #undef HAVE_LIBEVENT */
-
-/* Define to 1 if you have the header file. */
-#define HAVE_LIBINTL_H 1
-
-/* Define to 1 if you have the `pthread' library (-lpthread). */
-#define HAVE_LIBPTHREAD 1
-
-/* Define to 1 if you have the `rt' library (-lrt). */
-#define HAVE_LIBRT 1
-
-/* Define to 1 if you have the `socket' library (-lsocket). */
-/* #undef HAVE_LIBSOCKET */
-
-/* Define to 1 if you have the header file. */
-#define HAVE_LIMITS_H 1
-
-/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
- to 0 otherwise. */
-#define HAVE_MALLOC 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_MALLOC_H 1
-
-/* Define to 1 if you have the `memmove' function. */
-#define HAVE_MEMMOVE 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_MEMORY_H 1
-
-/* Define to 1 if you have the `memset' function. */
-#define HAVE_MEMSET 1
-
-/* Define to 1 if you have the `mkdir' function. */
-#define HAVE_MKDIR 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_NETDB_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_NETINET_IN_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_OPENSSL_RAND_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_OPENSSL_SSL_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_OPENSSL_X509V3_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_PTHREAD_H 1
-
-/* Define to 1 if the system has the type `ptrdiff_t'. */
-#define HAVE_PTRDIFF_T 1
-
-/* Define to 1 if your system has a GNU libc compatible `realloc' function,
- and to 0 otherwise. */
-#define HAVE_REALLOC 1
-
-/* Define to 1 if you have the `realpath' function. */
-#define HAVE_REALPATH 1
-
-/* Define to 1 if you have the `sched_get_priority_max' function. */
-#define HAVE_SCHED_GET_PRIORITY_MAX 1
-
-/* Define to 1 if you have the `sched_get_priority_min' function. */
-#define HAVE_SCHED_GET_PRIORITY_MIN 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SCHED_H 1
-
-/* Define to 1 if you have the `select' function. */
-#define HAVE_SELECT 1
-
-/* Define to 1 if you have the `socket' function. */
-#define HAVE_SOCKET 1
-
-/* Define to 1 if you have the `sqrt' function. */
-#define HAVE_SQRT 1
-
-/* Define to 1 if `stat' has the bug that it succeeds when given the
- zero-length file name argument. */
-/* #undef HAVE_STAT_EMPTY_STRING_BUG */
-
-/* Define to 1 if stdbool.h conforms to C99. */
-#define HAVE_STDBOOL_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STDDEF_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STDINT_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STDLIB_H 1
-
-/* Define to 1 if you have the `strchr' function. */
-#define HAVE_STRCHR 1
-
-/* Define to 1 if you have the `strdup' function. */
-#define HAVE_STRDUP 1
-
-/* Define to 1 if you have the `strerror' function. */
-#define HAVE_STRERROR 1
-
-/* Define to 1 if you have the `strerror_r' function. */
-#define HAVE_STRERROR_R 1
-
-/* Define to 1 if you have the `strftime' function. */
-#define HAVE_STRFTIME 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STRINGS_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_STRING_H 1
-
-/* Define to 1 if you have the `strstr' function. */
-#define HAVE_STRSTR 1
-
-/* Define to 1 if you have the `strtol' function. */
-#define HAVE_STRTOL 1
-
-/* Define to 1 if you have the `strtoul' function. */
-#define HAVE_STRTOUL 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_PARAM_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_POLL_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_RESOURCE_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_SELECT_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_SOCKET_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_STAT_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_TIME_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_TYPES_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_SYS_UN_H 1
-
-/* Define to 1 if you have that is POSIX.1 compatible. */
-#define HAVE_SYS_WAIT_H 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_UNISTD_H 1
-
-/* Define to 1 if you have the `vfork' function. */
-#define HAVE_VFORK 1
-
-/* Define to 1 if you have the header file. */
-/* #undef HAVE_VFORK_H */
-
-/* Define to 1 if you have the `vprintf' function. */
-#define HAVE_VPRINTF 1
-
-/* Define to 1 if you have the header file. */
-#define HAVE_WCHAR_H 1
-
-/* Define to 1 if `fork' works. */
-#define HAVE_WORKING_FORK 1
-
-/* Define to 1 if `vfork' works. */
-#define HAVE_WORKING_VFORK 1
-
-/* define if zlib is available */
-#define HAVE_ZLIB /**/
-
-/* Define to 1 if the system has the type `_Bool'. */
-/* #undef HAVE__BOOL */
-
-/* Possible value for SIGNED_RIGHT_SHIFT_IS */
-#define LOGICAL_RIGHT_SHIFT 2
-
-/* Define to 1 if `lstat' dereferences a symlink specified with a trailing
- slash. */
-#define LSTAT_FOLLOWS_SLASHED_SYMLINK 1
-
-/* Define to the sub-directory in which libtool stores uninstalled libraries.
- */
-#define LT_OBJDIR ".libs/"
-
-/* Name of package */
-#define PACKAGE "thrift"
-
-/* Define to the address where bug reports for this package should be sent. */
-#define PACKAGE_BUGREPORT ""
-
-/* Define to the full name of this package. */
-#define PACKAGE_NAME "thrift"
-
-/* Define to the full name and version of this package. */
-#define PACKAGE_STRING "thrift 0.9.1"
-
-/* Define to the one symbol short name of this package. */
-#define PACKAGE_TARNAME "thrift"
-
-/* Define to the home page for this package. */
-#define PACKAGE_URL ""
-
-/* Define to the version of this package. */
-#define PACKAGE_VERSION "0.9.1"
-
-/* Define as the return type of signal handlers (`int' or `void'). */
-#define RETSIGTYPE void
-
-/* Define to the type of arg 1 for `select'. */
-#define SELECT_TYPE_ARG1 int
-
-/* Define to the type of args 2, 3 and 4 for `select'. */
-#define SELECT_TYPE_ARG234 (fd_set *)
-
-/* Define to the type of arg 5 for `select'. */
-#define SELECT_TYPE_ARG5 (struct timeval *)
-
-/* Indicates the effect of the right shift operator on negative signed
- integers */
-#define SIGNED_RIGHT_SHIFT_IS 1
-
-/* If using the C implementation of alloca, define if you know the
- direction of stack growth for your system; otherwise it will be
- automatically deduced at runtime.
- STACK_DIRECTION > 0 => grows toward higher addresses
- STACK_DIRECTION < 0 => grows toward lower addresses
- STACK_DIRECTION = 0 => direction of growth unknown */
-/* #undef STACK_DIRECTION */
-
-/* Define to 1 if you have the ANSI C header files. */
-#define STDC_HEADERS 1
-
-/* Define to 1 if strerror_r returns char *. */
-#define STRERROR_R_CHAR_P 1
-
-/* Define to 1 if you can safely include both and . */
-#define TIME_WITH_SYS_TIME 1
-
-/* Define to 1 if your declares `struct tm'. */
-/* #undef TM_IN_SYS_TIME */
-
-/* Possible value for SIGNED_RIGHT_SHIFT_IS */
-#define UNKNOWN_RIGHT_SHIFT 3
-
-/* experimental --enable-boostthreads that replaces POSIX pthread by
- boost::thread */
-/* #undef USE_BOOST_THREAD */
-
-/* Version number of package */
-#define VERSION "0.9.1"
-
-/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
- `char[]'. */
-#define YYTEXT_POINTER 1
-
-/* Define for Solaris 2.5.1 so the uint32_t typedef from ,
- , or is not used. If the typedef were allowed, the
- #define below would cause a syntax error. */
-/* #undef _UINT32_T */
-
-/* Define for Solaris 2.5.1 so the uint64_t typedef from ,
- , or is not used. If the typedef were allowed, the
- #define below would cause a syntax error. */
-/* #undef _UINT64_T */
-
-/* Define for Solaris 2.5.1 so the uint8_t typedef from ,
- , or is not used. If the typedef were allowed, the
- #define below would cause a syntax error. */
-/* #undef _UINT8_T */
-
-/* Define to empty if `const' does not conform to ANSI C. */
-/* #undef const */
-
-/* Define to `__inline__' or `__inline' if that's what the C compiler
- calls it, or to nothing if 'inline' is not supported under any name. */
-#ifndef __cplusplus
-/* #undef inline */
-#endif
-
-/* Define to the type of a signed integer type of width exactly 16 bits if
- such a type exists and the standard includes do not define it. */
-/* #undef int16_t */
-
-/* Define to the type of a signed integer type of width exactly 32 bits if
- such a type exists and the standard includes do not define it. */
-/* #undef int32_t */
-
-/* Define to the type of a signed integer type of width exactly 64 bits if
- such a type exists and the standard includes do not define it. */
-/* #undef int64_t */
-
-/* Define to the type of a signed integer type of width exactly 8 bits if such
- a type exists and the standard includes do not define it. */
-/* #undef int8_t */
-
-/* Define to rpl_malloc if the replacement function should be used. */
-/* #undef malloc */
-
-/* Define to `int' if does not define. */
-/* #undef mode_t */
-
-/* Define to `long int' if does not define. */
-/* #undef off_t */
-
-/* Define to `int' if does not define. */
-/* #undef pid_t */
-
-/* Define to rpl_realloc if the replacement function should be used. */
-/* #undef realloc */
-
-/* Define to `unsigned int' if does not define. */
-/* #undef size_t */
-
-/* Define to `int' if does not define. */
-/* #undef ssize_t */
-
-/* Define to the type of an unsigned integer type of width exactly 16 bits if
- such a type exists and the standard includes do not define it. */
-/* #undef uint16_t */
-
-/* Define to the type of an unsigned integer type of width exactly 32 bits if
- such a type exists and the standard includes do not define it. */
-/* #undef uint32_t */
-
-/* Define to the type of an unsigned integer type of width exactly 64 bits if
- such a type exists and the standard includes do not define it. */
-/* #undef uint64_t */
-
-/* Define to the type of an unsigned integer type of width exactly 8 bits if
- such a type exists and the standard includes do not define it. */
-/* #undef uint8_t */
-
-/* Define as `fork' if `vfork' does not work. */
-/* #undef vfork */
-
-/* Define to empty if the keyword `volatile' does not work. Warning: valid
- code using `volatile' can become incorrect without. Disable with care. */
-/* #undef volatile */
-
-
-#endif
-
diff --git a/utils/thrift/thrift/thrift-config.h b/utils/thrift/thrift/thrift-config.h
index b1bcccba3..e60e4c1ca 100644
--- a/utils/thrift/thrift/thrift-config.h
+++ b/utils/thrift/thrift/thrift-config.h
@@ -1,24 +1,427 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
+/* lib/cpp/src/thrift/config.h. Generated from config.hin by configure. */
+/* config.hin. Generated from configure.ac by autoheader. */
-#ifdef _WIN32
-# include
-#else
-# include
+
+#ifndef CONFIG_H
+#define CONFIG_H
+
+
+/* Define if the AI_ADDRCONFIG symbol is unavailable */
+/* #undef AI_ADDRCONFIG */
+
+/* Possible value for SIGNED_RIGHT_SHIFT_IS */
+#define ARITHMETIC_RIGHT_SHIFT 1
+
+/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP
+ systems. This function is required for `alloca.c' support on those systems.
+ */
+/* #undef CRAY_STACKSEG_END */
+
+/* Define to 1 if using `alloca.c'. */
+/* #undef C_ALLOCA */
+
+/* Define to 1 if you have the `alarm' function. */
+#define HAVE_ALARM 1
+
+/* Define to 1 if you have `alloca', as a function or macro. */
+#define HAVE_ALLOCA 1
+
+/* Define to 1 if you have and it should be used (not on Ultrix).
+ */
+#define HAVE_ALLOCA_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* define if the Boost library is available */
+#define HAVE_BOOST /**/
+
+/* Define to 1 if you have the `bzero' function. */
+#define HAVE_BZERO 1
+
+/* Define to 1 if you have the `clock_gettime' function. */
+#define HAVE_CLOCK_GETTIME 1
+
+/* Define to 1 if you have the declaration of `strerror_r', and to 0 if you
+ don't. */
+#define HAVE_DECL_STRERROR_R 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */
+/* #undef HAVE_DOPRNT */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_FCNTL_H 1
+
+/* Define to 1 if you have the `fork' function. */
+#define HAVE_FORK 1
+
+/* Define to 1 if you have the `ftruncate' function. */
+#define HAVE_FTRUNCATE 1
+
+/* Define to 1 if you have the `gethostbyname' function. */
+#define HAVE_GETHOSTBYNAME 1
+
+/* Define to 1 if you have the `gettimeofday' function. */
+#define HAVE_GETTIMEOFDAY 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_INTTYPES_H 1
+
+/* define if libevent is available */
+/* #undef HAVE_LIBEVENT */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_LIBINTL_H 1
+
+/* Define to 1 if you have the `pthread' library (-lpthread). */
+#define HAVE_LIBPTHREAD 1
+
+/* Define to 1 if you have the `rt' library (-lrt). */
+#define HAVE_LIBRT 1
+
+/* Define to 1 if you have the `socket' library (-lsocket). */
+/* #undef HAVE_LIBSOCKET */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_LIMITS_H 1
+
+/* Define to 1 if your system has a GNU libc compatible `malloc' function, and
+ to 0 otherwise. */
+#define HAVE_MALLOC 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_MALLOC_H 1
+
+/* Define to 1 if you have the `memmove' function. */
+#define HAVE_MEMMOVE 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the `memset' function. */
+#define HAVE_MEMSET 1
+
+/* Define to 1 if you have the `mkdir' function. */
+#define HAVE_MKDIR 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_NETDB_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_NETINET_IN_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_RAND_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_SSL_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_X509V3_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if the system has the type `ptrdiff_t'. */
+#define HAVE_PTRDIFF_T 1
+
+/* Define to 1 if your system has a GNU libc compatible `realloc' function,
+ and to 0 otherwise. */
+#define HAVE_REALLOC 1
+
+/* Define to 1 if you have the `realpath' function. */
+#define HAVE_REALPATH 1
+
+/* Define to 1 if you have the `sched_get_priority_max' function. */
+#define HAVE_SCHED_GET_PRIORITY_MAX 1
+
+/* Define to 1 if you have the `sched_get_priority_min' function. */
+#define HAVE_SCHED_GET_PRIORITY_MIN 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SCHED_H 1
+
+/* Define to 1 if you have the `select' function. */
+#define HAVE_SELECT 1
+
+/* Define to 1 if you have the `socket' function. */
+#define HAVE_SOCKET 1
+
+/* Define to 1 if you have the `sqrt' function. */
+#define HAVE_SQRT 1
+
+/* Define to 1 if `stat' has the bug that it succeeds when given the
+ zero-length file name argument. */
+/* #undef HAVE_STAT_EMPTY_STRING_BUG */
+
+/* Define to 1 if stdbool.h conforms to C99. */
+#define HAVE_STDBOOL_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDDEF_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the `strchr' function. */
+#define HAVE_STRCHR 1
+
+/* Define to 1 if you have the `strdup' function. */
+#define HAVE_STRDUP 1
+
+/* Define to 1 if you have the `strerror' function. */
+#define HAVE_STRERROR 1
+
+/* Define to 1 if you have the `strerror_r' function. */
+#define HAVE_STRERROR_R 1
+
+/* Define to 1 if you have the `strftime' function. */
+#define HAVE_STRFTIME 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the `strstr' function. */
+#define HAVE_STRSTR 1
+
+/* Define to 1 if you have the `strtol' function. */
+#define HAVE_STRTOL 1
+
+/* Define to 1 if you have the `strtoul' function. */
+#define HAVE_STRTOUL 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_PARAM_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_POLL_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_RESOURCE_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_SELECT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_SOCKET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_UN_H 1
+
+/* Define to 1 if you have that is POSIX.1 compatible. */
+#define HAVE_SYS_WAIT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the `vfork' function. */
+#define HAVE_VFORK 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_VFORK_H */
+
+/* Define to 1 if you have the `vprintf' function. */
+#define HAVE_VPRINTF 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_WCHAR_H 1
+
+/* Define to 1 if `fork' works. */
+#define HAVE_WORKING_FORK 1
+
+/* Define to 1 if `vfork' works. */
+#define HAVE_WORKING_VFORK 1
+
+/* define if zlib is available */
+#define HAVE_ZLIB /**/
+
+/* Define to 1 if the system has the type `_Bool'. */
+/* #undef HAVE__BOOL */
+
+/* Possible value for SIGNED_RIGHT_SHIFT_IS */
+#define LOGICAL_RIGHT_SHIFT 2
+
+/* Define to 1 if `lstat' dereferences a symlink specified with a trailing
+ slash. */
+#define LSTAT_FOLLOWS_SLASHED_SYMLINK 1
+
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+ */
+#define LT_OBJDIR ".libs/"
+
+/* Name of package */
+#define PACKAGE "thrift"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "thrift"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "thrift 0.9.1"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "thrift"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "0.9.1"
+
+/* Define as the return type of signal handlers (`int' or `void'). */
+#define RETSIGTYPE void
+
+/* Define to the type of arg 1 for `select'. */
+#define SELECT_TYPE_ARG1 int
+
+/* Define to the type of args 2, 3 and 4 for `select'. */
+#define SELECT_TYPE_ARG234 (fd_set *)
+
+/* Define to the type of arg 5 for `select'. */
+#define SELECT_TYPE_ARG5 (struct timeval *)
+
+/* Indicates the effect of the right shift operator on negative signed
+ integers */
+#define SIGNED_RIGHT_SHIFT_IS 1
+
+/* If using the C implementation of alloca, define if you know the
+ direction of stack growth for your system; otherwise it will be
+ automatically deduced at runtime.
+ STACK_DIRECTION > 0 => grows toward higher addresses
+ STACK_DIRECTION < 0 => grows toward lower addresses
+ STACK_DIRECTION = 0 => direction of growth unknown */
+/* #undef STACK_DIRECTION */
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Define to 1 if strerror_r returns char *. */
+#define STRERROR_R_CHAR_P 1
+
+/* Define to 1 if you can safely include both and . */
+#define TIME_WITH_SYS_TIME 1
+
+/* Define to 1 if your declares `struct tm'. */
+/* #undef TM_IN_SYS_TIME */
+
+/* Possible value for SIGNED_RIGHT_SHIFT_IS */
+#define UNKNOWN_RIGHT_SHIFT 3
+
+/* experimental --enable-boostthreads that replaces POSIX pthread by
+ boost::thread */
+/* #undef USE_BOOST_THREAD */
+
+/* Version number of package */
+#define VERSION "0.9.1"
+
+/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a
+ `char[]'. */
+#define YYTEXT_POINTER 1
+
+/* Define for Solaris 2.5.1 so the uint32_t typedef from ,
+ , or is not used. If the typedef were allowed, the
+ #define below would cause a syntax error. */
+/* #undef _UINT32_T */
+
+/* Define for Solaris 2.5.1 so the uint64_t typedef from ,
+ , or is not used. If the typedef were allowed, the
+ #define below would cause a syntax error. */
+/* #undef _UINT64_T */
+
+/* Define for Solaris 2.5.1 so the uint8_t typedef from ,
+ , or is not used. If the typedef were allowed, the
+ #define below would cause a syntax error. */
+/* #undef _UINT8_T */
+
+/* Define to empty if `const' does not conform to ANSI C. */
+/* #undef const */
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+ calls it, or to nothing if 'inline' is not supported under any name. */
+#ifndef __cplusplus
+/* #undef inline */
#endif
+
+/* Define to the type of a signed integer type of width exactly 16 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef int16_t */
+
+/* Define to the type of a signed integer type of width exactly 32 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef int32_t */
+
+/* Define to the type of a signed integer type of width exactly 64 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef int64_t */
+
+/* Define to the type of a signed integer type of width exactly 8 bits if such
+ a type exists and the standard includes do not define it. */
+/* #undef int8_t */
+
+/* Define to rpl_malloc if the replacement function should be used. */
+/* #undef malloc */
+
+/* Define to `int' if does not define. */
+/* #undef mode_t */
+
+/* Define to `long int' if does not define. */
+/* #undef off_t */
+
+/* Define to `int' if does not define. */
+/* #undef pid_t */
+
+/* Define to rpl_realloc if the replacement function should be used. */
+/* #undef realloc */
+
+/* Define to `unsigned int' if does not define. */
+/* #undef size_t */
+
+/* Define to `int' if does not define. */
+/* #undef ssize_t */
+
+/* Define to the type of an unsigned integer type of width exactly 16 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef uint16_t */
+
+/* Define to the type of an unsigned integer type of width exactly 32 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef uint32_t */
+
+/* Define to the type of an unsigned integer type of width exactly 64 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef uint64_t */
+
+/* Define to the type of an unsigned integer type of width exactly 8 bits if
+ such a type exists and the standard includes do not define it. */
+/* #undef uint8_t */
+
+/* Define as `fork' if `vfork' does not work. */
+/* #undef vfork */
+
+/* Define to empty if the keyword `volatile' does not work. Warning: valid
+ code using `volatile' can become incorrect without. Disable with care. */
+/* #undef volatile */
+
+
+#endif
+
diff --git a/utils/thrift/thrift/windows/config.h b/utils/thrift/thrift/windows/config.h
deleted file mode 100644
index 0555e079f..000000000
--- a/utils/thrift/thrift/windows/config.h
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-#ifndef _THRIFT_WINDOWS_CONFIG_H_
-#define _THRIFT_WINDOWS_CONFIG_H_ 1
-
-#if defined(_MSC_VER) && (_MSC_VER > 1200)
-#pragma once
-#endif // _MSC_VER
-
-#ifndef _WIN32
-#error This is a MSVC header only.
-#endif
-
-// use std::thread in MSVC11 (2012) or newer
-#if _MSC_VER >= 1700
-# define USE_STD_THREAD 1
-// otherwise use boost threads
-#else
-# define USE_BOOST_THREAD 1
-#endif
-
-#ifndef TARGET_WIN_XP
-# define TARGET_WIN_XP 1
-#endif
-
-#if TARGET_WIN_XP
-# ifndef WINVER
-# define WINVER 0x0501
-# endif
-# ifndef _WIN32_WINNT
-# define _WIN32_WINNT 0x0501
-# endif
-#endif
-
-#ifndef _WIN32_WINNT
-# define _WIN32_WINNT 0x0601
-#endif
-
-#pragma warning(disable: 4996) // Deprecated posix name.
-
-#define VERSION "1.0.0-dev"
-#define HAVE_GETTIMEOFDAY 1
-#define HAVE_SYS_STAT_H 1
-
-#ifdef HAVE_STDINT_H
-# include
-#else
-# include
-
-typedef boost::int64_t int64_t;
-typedef boost::uint64_t uint64_t;
-typedef boost::int32_t int32_t;
-typedef boost::uint32_t uint32_t;
-typedef boost::int16_t int16_t;
-typedef boost::uint16_t uint16_t;
-typedef boost::int8_t int8_t;
-typedef boost::uint8_t uint8_t;
-#endif
-
-#include
-#include
-#include
-#include
-#include
-#include
-
-// windows
-#include
-#include
-#pragma comment(lib, "Ws2_32.lib")
-#pragma comment(lib, "advapi32.lib") //For security APIs in TPipeServer
-
-#endif // _THRIFT_WINDOWS_CONFIG_H_
diff --git a/writeengine/bulk/bulkload.py b/writeengine/bulk/bulkload.py
deleted file mode 100644
index 9a6ab0e06..000000000
--- a/writeengine/bulk/bulkload.py
+++ /dev/null
@@ -1,299 +0,0 @@
-#!/usr/bin/python
-##
-## Bulkloader script by Martin Thomas
-##
-
-import os, sys, glob, shutil, xml.dom.minidom
-import getopt
-import logging
-import time
-
-logger = logging.getLogger()
-shdlr = logging.StreamHandler()
-fhdlr = logging.FileHandler(filename='bulkload.log' )
-formatter = logging.Formatter('%(asctime)s:%(levelname)s: %(message)s')
-shdlr.setFormatter(formatter)
-fhdlr.setFormatter(formatter)
-logger.addHandler(shdlr)
-logger.addHandler(fhdlr)
-
-## only report INFO or higher - change to WARNING to silence all logging
-logger.setLevel(logging.INFO)
-
-
-def usage():
- print """
-
- Bulkload.py is intended to automate the manual steps required to load the database and build indexes from scratch.
-
- - ipcs-pat will be built if missing
- - cpimport will be removed and rebuilt
- - PrimProc will be stopped and started
- - shared memory sgements wil be removed using ipcs-pat
- - database files will be removed
- - dbgen will be run with option 5
- - oid files and job files will be copied to correct locations
- - column data will be parsed and loaded using Job 299
- - index data will be exported, sorted and loaded using Job 300
-
- Options:
- -w or --wedir= : Specify the write engine branch to use instead of the default trunk
- -n or --nocache= : Specify either col or idx and the -c flag will NOT be sent to cpimport
- -u or --usage : Usage message
-
- Example:
- bulkload.py -w/home/adevelop/genii/we1.1 --nocache=idx
- Load the database using the we1.1 branch for writeengine and do not use cache when building indexes
-
- THIS SPACE LEFT INTENTIONALLY BLANK
- """
-
-def find_paths():
-
- """Find DBRoot and BulkRoot."""
- try:
- config_file = os.environ['COLUMNSTORE_CONFIG_FILE']
- except KeyError:
- try:
- logger.info("Environment variable COLUMNSTORE_CONFIG_FILE not set, looking for system Columnstore.xml")
- config_file = '/usr/local/mariadb/columnstore/etc/Columnstore.xml'
- os.lstat(config_file)
- except:
- logger.error('No config file available')
- sys.exit('No config file available')
- try:
- xmldoc = xml.dom.minidom.parse(config_file)
- bulk_node = xmldoc.getElementsByTagName('BulkRoot')[0]
- db_node = xmldoc.getElementsByTagName('DBRoot1')[0]
- bulk_dir = bulk_node.childNodes[0].nodeValue
- data_dir = db_node.childNodes[0].nodeValue
-
- except Exception, e:
- logger.error('Error parsing config file')
- logger.error(e)
- sys.exit('Error parsing config file')
-
- return (bulk_dir, data_dir)
-
-def check_dirs(bulkroot, dbroot):
-
- problem = 0
- res = 0
- reqd_dirs = {
- os.getenv('HOME')+'/genii' : "No genii directory found (contains tools required to continue) (%s)",
- bulkroot: "Bulkroot specified as %s but not found",
- bulkroot+'/job': "No job directory found - needed to store Job xml files (looked in %s)",
- bulkroot+'/data/import': "No data/import directory found - expected %s to hold data to be loaded",
- bulkroot+'/log': "No data/log directory found - expected %s to log into",
- dbroot : "DBroot specified as %s but not found"
- }
- for dir in reqd_dirs.keys():
- try:
- res = os.lstat(dir)
- except:
- problem = 1
- logger.error(reqd_dirs[dir]%dir)
-
- if problem:
- sys.exit(1)
-
-def fix_hwm(job_file):
-
- """Find hwm in xml file and change to 0"""
-
- import re
-
- src_file = open(job_file, 'r')
- dst_file = open(job_file+'.tmp', 'w')
-
- rep = re.compile('hwm="1"')
-
- for line in src_file:
- line = rep.sub('hwm="0"', line)
- dst_file.write(line)
- # use os.rename instead of shutil.move to avoid problems traversing devices
- os.rename(job_file+'.tmp', job_file)
-
-def find_indexes(job_file):
-
- """Find index definitions in job_file and return list of files to sort"""
-
- index_files = []
- try: # try because we may have an old version of python
- xmldoc = xml.dom.minidom.parse(job_file)
-
- for index_node in xmldoc.getElementsByTagName('Index'):
- index_files.append(index_node.getAttribute('mapName'))
- except:
- import re
- f = open(job_file)
- for line in f.read():
- b =re.search('mapName="(CPL_[0-9A-Z_]+)"', line)
- try: # try because not every line will match
- index_files.append(b.group(1))
- except: pass
-
- return index_files
-
-def exec_cmd(cmd, args):
- """Execute command using subprocess module or if that fails,
- use os.system
- """
-
- try:
- import subprocess
-
- try:
- retcode = call(cmd + " "+args, shell=True)
- if retcode < 0:
- print >>sys.stderr, "Child was terminated by signal", -retcode
- sys.exit(-1)
-
- else:
- print >>sys.stderr, "Child returned", retcode
-
- except OSError, e:
-
- print >>sys.stderr, "Execution failed:", e
- sys.exit(-1)
- except:
- logger.info ('Old version of Python - subprocess not available, falling back to os.system')
- logger.info ('Executing: '+cmd+' '+args)
- res = os.system(cmd+' '+args)
- if res:
- logger.error('Bad return code %i from %s'%(res, cmd))
- sys.exit( res )
-
-
-def build_tool(tool):
- """
- Use the tool dictionary to determine if required tool exists
- and build if not
- """
-
- if not os.path.exists(tool['path']+tool['tool']):
- logger.warn ("Building %s before continuing"%tool['tool'])
- curdir=os.getcwd()
- os.chdir(tool['path'])
- exec_cmd(tool['builder'], tool['args'])
- os.chdir(curdir)
-
-def main():
- """
- Bulk load the database..
- Check that we can write OIDfiles, that all required tools exist,
- clean up old files, sort the index inserts and generally rock and roll
- """
- start_dir = curdir=os.getcwd() # remember where we started
-
- if not os.environ.has_key('LD_LIBRARY_PATH'):
- logger.info('No environment variable LD_LIBRARY_PATH')
- else:
- if len(os.getenv('LD_LIBRARY_PATH'))<5:
- logger.info('Suspicous LD_LIBRARY_PATH: %s'%os.getenv('LD_LIBRARY_PATH'))
-
- #-- figure out paths
- home = os.getenv('HOME')
- genii = home+'/genii'
- cache = {}
- cache['idx'] = '-c'
- cache['col'] = '-c'
-
-#-- allow us to specify a write engine branch
- opts, args = getopt.getopt(sys.argv[1:], 'w:n:u', ['wedir=', 'nocache=', 'usage'])
- wedir = genii+'/writeengine'
- for opt, arg in opts:
- if opt =='-w' or opt =='--wedir':
- wedir = arg
-
- if opt == '-n' or opt == '--nocache':
- if (arg=='idx' or arg=='col'):
- cache[arg] = ''
- logger.info("No cache for %s"% arg)
-
- if opt == '-u' or opt == '--usage':
- usage()
- sys.exit()
-
- logger.info("Using writengine at %s"%wedir)
-
- (bulkroot, dbroot) = find_paths()
-
- logger.info ("Bulkroot: %s \tDBRoot: %s\n"%(bulkroot, dbroot))
-
- check_dirs(bulkroot, dbroot)
-
- if len(glob.glob(bulkroot+'/data/import/*tbl')) == 0:
- sys.exit("No files for import found in BulkRoot: %s"%(bulkroot))
-
- if len(glob.glob(dbroot+'/000.dir'))==0:
- logger.info("No files found in DBRoot: %s (not fatal)"%dbroot)
-
-## force rebuild cpimport and build ipcs-pat if required
-
- build_tool({'path':genii+'/versioning/BRM/',
- 'tool':'ipcs-pat',
- 'builder':'make', 'args':'tools'})
-
- build_tool({'path':wedir+'/bulk/',
- 'tool':'cpimport',
- 'builder':'make', 'args':'clean'})
- try:
- exec_cmd('rm -f', wedir+'/bulk/cpimport')
- except:
- pass
-
- try:
- os.lstat(start_dir+'/cpimport') # look in local directory first
- except:
- build_tool({'path':wedir+'/bulk/',
- 'tool':'cpimport',
- 'builder':'make', 'args':'cpimport'})
-
-
-## clean up before starting
-## remove old db files, removed old temp files, remove shared memory segments,
-## kill old PrimProc and start new one
-
- logger.info ("Removing old DB files")
- exec_cmd('rm -fr ', dbroot+'/000.dir')
-
- logger.info ("Removing old temp files")
- exec_cmd('rm -fr ', bulkroot+'/data/import/*.idx.txt')
-
- logger.info ("Removing old process files")
- exec_cmd('rm -fr ', bulkroot+'/process/*.*')
-
- logger.info("Killing primProc")
- os.system('killall -q -u $USER PrimProc')
-
- logger.info ("kill controllernode and workernode")
- exec_cmd(genii+'/export/bin/dbrm', "stop ")
-
- time.sleep(2);
- logger.info ("Removing shared memory segments")
- exec_cmd(genii+'/versioning/BRM/ipcs-pat', '-d')
-
- logger.info("Starting controllernode workernode")
- exec_cmd(genii+'/export/bin/dbrm', "start ")
-
- logger.info("Starting primProc")
- exec_cmd(genii+'/export/bin/PrimProc', "> primproc.log &")
-
-## run dbbuilder - add yes command at front to automatically answer questions
- logger.info ("Building db and indexes (no data inserted)")
- exec_cmd('yes | '+genii+'/tools/dbbuilder/dbbuilder', ' 5')
-
- logger.info ("Relocating OID files")
-
- for xmlfile in glob.glob('./Job*xml'):
- logger.info ("Copying %s to %s\n"%(xmlfile, bulkroot+'/job'))
- # use os.rename instead of shutil.move to avoid problems traversing devices
- os.rename(xmlfile, bulkroot+'/job/'+xmlfile)
-
- logger.info("Using cpimport at %s"%(wedir+'/bulk/cpimport'))
- exec_cmd('time '+wedir+'/bulk/cpimport', '-j 299 ')
- exec_cmd(wedir+'/bulk/cpimport', '-c -j 300 ' )
-
-## the following line allows either interactive use or module import
-if __name__=="__main__": main()
diff --git a/writeengine/bulk/bulkloadp.sh b/writeengine/bulk/bulkloadp.sh
deleted file mode 100755
index 80015e12b..000000000
--- a/writeengine/bulk/bulkloadp.sh
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/bin/bash
-
-#This is the procedure for running bulkload using cpimport program
-#Usage of this program :
-#The necessary input parameter is the schema name
-#For example: bulkload.sh TPCH
-
-#A table name and a Job ID can be entered by user when it is prompted or they can be skipped by hitting enter key
-#When the table name is skipped, ALL of the columns and index in ALL of the tables in the schema will be loaded
-
-#When table name is entered, All of the columns and indexes in the entered table will be loaded
-#Job ID will determine the names of the two xml files. For example, job id 100 will generate Job_100.xml for columns and Job_101 for index xml file. Job id for index xml file is the entered job id +1
-#if the job id is skipped, the default job ids are 299 and 300 for column and index files
-#There are two xml files will be generated which reside in bulkroot directory under subdirectory job
-#For example, the job directory may look like /usr/local/mariadb/columnstore/test/bulk/job
-
-# Set up a default search path.
-PROG_NAME=$(basename $0)
-SUFFIX=.tbl
-TABLENAME=""
-while getopts 't:j:e:s:d:p:n:u:h' OPTION
-do
- case ${OPTION} in
- s) Schema=${OPTARG};;
- t) TABLENAME=${OPTARG};;
- j) JOBID=${OPTARG};;
- e) MAXERROR=${OPTARG};;
- p) DESC=${OPTARG};;
- d) DELIMITER=${OPTARG};;
- n) NAME=${OPTARG};;
- u) USER=${OPTARG};;
- h) echo "Options: ${PROG_NAME} -s schema -j jobid [-t TableName -e max_error_row -p description -d delimiter -n name -u user]"
- exit 2;;
- \?) echo "Options: ${PROG_NAME} -s schema -j jobid [-t TableName -e max_error_row -s description -d delimiter -n name -u user]"
- exit 2;;
- esac
-done
-
-#generate column xml file
-echo "MAXERROR in $PROG_NAME =" $MAXERROR
-echo "JOBID in $PROG_NAME =" $JOBID
-echo "Schema is " $Schema
-echo "DESC is " $DESC
-echo "DELIMITER =" $DELIMITER
-echo "TABLENAME is " $TABLENAME
-echo "NAME is " $NAME
-
-if [ -n "$TABLENAME" ]; then
- ./colxml $Schema -t $TABLENAME -j $JOBID -d $DELIMITER -s "$DESC" -e $MAXERROR -n "$NAME" -u $USER
-if [ "$?" <> "0" ]; then
- echo "Error in colxml !" 1>&2
- exit 1
-fi
-command="colxml $Schema -t $TABLENAME -j $JOBID -d $DELIMITER -s \"$DESC\" -e $MAXERROR -n \"$NAME\" -u \"$USER\" "
- echo $command
-else
- ./colxml $Schema -j $JOBID -d $DELIMITER -s "$DESC" -e $MAXERROR -n "$NAME" -u $USER
-if [ "$?" <> "0" ]; then
- echo "Error in colxml !" 1>&2
- exit 1
-fi
- command="colxml $Schema -j $JOBID -d "$DELIMITER" -s \"$DESC\" -e $MAXERROR -n \"$NAME\" -u \"$USER\" "
- echo $command
-fi
-
-#generate index xml file
-DESC="table index definition"
-NAME="index definitions for tables in $Schema"
-let "JOBID2 = JOBID+1"
-echo "DEFAULT INDEX JOB ID is " $JOBID2
-if [ -n "$TABLENAME" ]; then
- ./indxml $Schema -t $TABLENAME -j $JOBID2 -s "$DESC" -e $MAXERROR -n "$NAME" -u $USER
-if [ "$?" <> "0" ]; then
- echo "Error in indxml !" 1>&2
- exit 1
-fi
-
- command="indxml $Schema -t $TABLENAME -j $JOBID2 -s \"$DESC\" -e $MAXERROR -n \"$NAME\" -u \"$USER\" "
- echo $command
-
-else
- ./indxml $Schema -j $JOBID2 -s "$DESC" -e $MAXERROR -n "$NAME" -u $USER
-if [ "$?" <> "0" ]; then
- echo "Error in colxml !" 1>&2
- exit 1
-fi
-
- command="indxml $Schema -j $JOBID2 -s \"$DESC\" -e $MAXERROR -n \"$NAME\" -u \"$USER\" "
- echo $command
-fi
-#get bulkroot
-if [ -n "$CALPONT_CONFIG_FILE" ]; then
- echo "CALPONT_CONFIG_FILE=" $CALPONT_CONFIG_FILE
-elif [ -z "$CALPONT_CONFIG_FILE"]; then
- CALPONT_CONFIG_FILE="/usr/local/mariadb/columnstore/etc/Columnstore.xml"
- echo "CALPONT_CONFIG_FILE=" $CALPONT_CONFIG_FILE
-else
- CALPONT_CONFIG_FILE="/usr/local/mariadb/columnstore/etc/Columnstore.xml"
- echo "CALPONT_CONFIG_FILE=" $CALPONT_CONFIG_FILE
-fi
-
-awk '/BulkRoot/ { sub(//,"",$0); sub(/<\/BulkRoot>/,"",$0); sub(/" "/,"",$0);print $0 > "tmp.txt"}' $CALPONT_CONFIG_FILE
-sed -e 's/ *//g' tmp.txt > out.txt
-
-BulkRoot=$(cat out.txt)
-echo "BulkRoot=" $BulkRoot
-rm -rf out.txt tmp.txt
-
-#bulk load column files
-./cpimport -j $JOBID
-command="cpimport -j $JOBID"
-echo $command
-#bulk load parallel index files
-#./splitidx -j $JOBID2
-#IDX_SHELL_SCRIPT="$BulkRoot/process/Job_$JOBID2.sh"
-#chmod +x $IDX_SHELL_SCRIPT
-#echo " run parallel loading $IDX_SHELL_SCRIPT"
-#$IDX_SHELL_SCRIPT
-
-
-
diff --git a/writeengine/bulk/checkidx.py b/writeengine/bulk/checkidx.py
deleted file mode 100755
index be22a674b..000000000
--- a/writeengine/bulk/checkidx.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/python
-
-import os, sys, glob, shutil, xml.dom.minidom
-
-def find_paths():
-
- """Find DBRoot and BulkRoot."""
- try:
- config_file = os.environ['COLUMNSTORE_CONFIG_FILE']
- except KeyError:
- try:
- config_file = '/usr/local/mariadb/columnstore/etc/Columnstore.xml'
- os.lstat(config_file)
- except:
- sys.exit('No config file available')
-
-
- xmldoc = xml.dom.minidom.parse(config_file)
- bulk_node = xmldoc.getElementsByTagName('BulkRoot')[0]
- db_node = xmldoc.getElementsByTagName('DBRoot')[0]
-
- bulk_dir = bulk_node.childNodes[0].nodeValue
- data_dir = db_node.childNodes[0].nodeValue
-
- return (bulk_dir, data_dir)
-
-
-def validate_indexes(job_file):
- index_files = []
- xmldoc = xml.dom.minidom.parse(job_file)
-
- for index_node in xmldoc.getElementsByTagName('Index'):
- curTreeOid = index_node.getAttribute('iTreeOid')
- curListOid = index_node.getAttribute('iListOid')
- curMapOid = index_node.getAttribute('mapOid')
- #curIdxCmdArg = ' -t ' + curTreeOid + ' -l ' + curListOid + ' -v -c ' + curMapOid + ' > idxCol_' + curMapOid+'.out'
- curIdxCmdArg = ' -t %s -l %s -v -c %s > idxCol_%s.out' % (curTreeOid, curListOid, curMapOid, curMapOid)
- index_files.append( curIdxCmdArg )
-
- return index_files
-
-def exec_cmd(cmd, args):
- """Execute command using subprocess module or if that fails,
- use os.system
- """
-
- try:
- import subprocess
-
- try:
- retcode = call(cmd + " "+args, shell=True)
- if retcode < 0:
- print >>sys.stderr, "Child was terminated by signal", -retcode
- sys.exit(-1)
-
- else:
- print >>sys.stderr, "Child returned", retcode
-
- except OSError, e:
-
- print >>sys.stderr, "Execution failed:", e
- sys.exit(-1)
- except:
- res = os.system(cmd+' '+args)
- if res:
- sys.exit( res )
-
-
-
-def main():
- """
- Validate indexes..
- """
-
- if len(os.getenv('LD_LIBRARY_PATH'))<5:
- print 'Suspicous LD_LIBRARY_PATH: %s'%os.getenv('LD_LIBRARY_PATH')
-
- home = os.getenv('HOME')
- genii = home+'/genii'
-
- (bulkroot, dbroot) = find_paths()
-
- if len(glob.glob(bulkroot+'/job/Job_300.xml')) == 0:
- sys.exit("No Job_300.xml exist ")
-
- indexes = validate_indexes(bulkroot+'/job/Job_300.xml')
- for idxCmdArg in indexes:
- print idxCmdArg
- exec_cmd( genii + '/tools/evalidx/evalidx', idxCmdArg )
-
-
-## the following line allows either interactive use or module import
-if __name__=="__main__": main()
diff --git a/writeengine/bulk/cpimport.sh b/writeengine/bulk/cpimport.sh
deleted file mode 100755
index 2050fc6cf..000000000
--- a/writeengine/bulk/cpimport.sh
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/bin/bash
-
-#This is the procedure for running bulkload using cpimport program
-#Usage of this program :
-#The necessary input parameter is the schema name
-#For example: bulkload.sh TPCH
-
-#A table name and a Job ID can be entered by user when it is prompted or they can be skipped by hitting enter key
-#When the table name is skipped, ALL of the columns and index in ALL of the tables in the schema will be loaded
-
-#When table name is entered, All of the columns and indexes in the entered table will be loaded
-#Job ID will determine the names of the two xml files. For example, job id 100 will generate Job_100.xml for columns and Job_101 for index xml file. Job id for index xml file is the entered job id +1
-#if the job id is skipped, the default job ids are 299 and 300 for column and index files
-#There are two xml files will be generated which reside in bulkroot directory under subdirectory job
-#For example, the job directory may look like /usr/local/mariadb/columnstore/test/bulk/job
-
-# Set up a default search path.
-
-#echo "This is Script name " $0
-PROG_NAME=$(basename $0)
-
-USERNAME=`grep "^${USER}:" /etc/passwd | cut -d: -f5`
-JOBID=""
-TABLENAME=""
-Schema=""
-DELIMITER="|"
-MAXERROR=10
-FORMAT=CSV
-DESC="table columns definition"
-NAME="table columns definition"
-
-
-while getopts 't:j:e:s:d:p:n:hu' OPTION
-do
- case ${OPTION} in
- s) Schema=${OPTARG};;
- t) TABLENAME=${OPTARG};;
- j) JOBID=${OPTARG};;
- e) MAXERROR=${OPTARG};;
- p) DESC=${OPTARG};;
- d) DELIMITER=${OPTARG};;
- n) NAME=${OPTARG};;
- h) echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -e max_error_row -p description -d delimiter -n name ]"
- exit 2;;
- u) echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -e max_error_row -p description -d delimiter -n name ]"
- exit 2;;
- \?) echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -e max_error_row -p description -d delimiter -n name ]"
- exit 2;;
- esac
-done
-
-if [ -n "$Schema" ]; then
- echo "Schema is " $Schema
-else
- echo "Error using the script, a schema is needed! "
- echo "usage as follows: "
- echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -p description -d delimiter -e max_error_rows -n name ]"
- echo "PLEASE ONLY INPUT SCHEMA NAME:"
- read Schema
- if [ -n "$Schema" ]; then
- echo "Schema is " $Schema
- else
- echo "Error using the script, a schema is needed! "
- echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -p description -d delimiter -e max_error_rows -n name ]"
- echo "Try again! Goodbye!"
- exit 2;
- fi
-fi
-NAME="column definitions for tables in $Schema"
-
-if [ -n "$JOBID" ]; then
- echo "INPUT JOB ID is " $JOBID
-else
- echo "Error using the script, a jobid is needed! "
- echo "PLEASE INPUT jobid:"
- read JOBID
- if [ -n "$JOBID" ]; then
- echo "JOBID is " $JOBID
- else
- echo "Error using the script, a jobid is needed! "
- echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -s description -d delimiter -e max_error_rows -n name ]"
- echo "Try again! Goodbye!"
- exit 2;
- fi
-fi
-################################################################################
-
-if [ -n "$TABLENAME" ]; then
- ./bulkloadp.sh -e $MAXERROR -s $Schema -t "$TABLENAME" -j $JOBID -p "$DESC" -d "$DELIMITER" -n "$NAME" -u $USER
-
-else
- ./bulkloadp.sh -e $MAXERROR -s $Schema -j $JOBID -d "$DELIMITER" -p "$DESC" -n "$NAME" -u $USER
-fi
diff --git a/writeengine/bulk/dbload_tmplate.sh b/writeengine/bulk/dbload_tmplate.sh
deleted file mode 100755
index 9f050a469..000000000
--- a/writeengine/bulk/dbload_tmplate.sh
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/bin/bash
-
-#This is the procedure for running bulkload using cpimport program
-#Usage of this program :
-#The necessary input parameter is the schema name
-#For example: bulkload.sh TPCH
-
-#A table name and a Job ID can be entered by user when it is prompted or they can be skipped by hitting enter key
-#When the table name is skipped, ALL of the columns and index in ALL of the tables in the schema will be loaded
-
-#When table name is entered, All of the columns and indexes in the entered table will be loaded
-#Job ID will determine the names of the two xml files. For example, job id 100 will generate Job_100.xml for columns and Job_101 for index xml file. Job id for index xml file is the entered job id +1
-#if the job id is skipped, the default job ids are 299 and 300 for column and index files
-#There are two xml files will be generated which reside in bulkroot directory under subdirectory job
-#For example, the job directory may look like /usr/local/mariadb/columnstore/test/bulk/job
-
-# Set up a default search path.
-PATH="$HOME/genii/export/bin:.:/sbin:/usr/sbin:/bin:/usr/bin:/usr/X11R6/bin"
-export PATH
-
-#echo "This is Script name " $0
-PROG_NAME=$(basename $0)
-
-USERNAME=`grep "^${USER}:" /etc/passwd | cut -d: -f5`
-JOBID=""
-TABLENAME=""
-Schema=""
-DELIMITER="|"
-MAXERROR=10
-FORMAT=CSV
-DESC="table columns definition"
-NAME="table columns definition"
-
-
-while getopts 't:j:e:s:d:p:n:hu' OPTION
-do
- case ${OPTION} in
- s) Schema=${OPTARG};;
- t) TABLENAME=${OPTARG};;
- j) JOBID=${OPTARG};;
- e) MAXERROR=${OPTARG};;
- p) DESC=${OPTARG};;
- d) DELIMITER=${OPTARG};;
- n) NAME=${OPTARG};;
- h) echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -e max_error_row -p description -d delimiter -n name ]"
- exit 2;;
- u) echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -e max_error_row -p description -d delimiter -n name ]"
- exit 2;;
- \?) echo "Usage: ${PROG_NAME} -s schema -j jobid [-t TableName -e max_error_row -p description -d delimiter -n name ]"
- exit 2;;
- esac
-done
-
-if [ -n "$Schema" ]; then
- echo "Schema is " $Schema
-else
- echo "Error using the script, a schema is needed! "
- echo "usage as follows: "
- echo "Usage: ${PROG_NAME} Schema -j jobid [-t TableName -p description -d delimiter -e max_error_rows -n name ]"
- echo "PLEASE ONLY INPUT SCHEMA NAME:"
- read Schema
- if [ -n "$Schema" ]; then
- echo "Schema is " $Schema
- else
- echo "Error using the script, a schema is needed! "
- echo "Usage: ${PROG_NAME} Schema -j jobid [-t TableName -p description -d delimiter -e max_error_rows -n name ]"
- echo "Try again! Goodbye!"
- exit 2;
- fi
-fi
-NAME="column definitions for tables in $Schema"
-
-if [ -n "$JOBID" ]; then
- echo "INPUT JOB ID is " $JOBID
-else
- echo "Error using the script, a jobid is needed! "
- echo "PLEASE INPUT jobid:"
- read JOBID
- if [ -n "$JOBID" ]; then
- echo "JOBID is " $JOBID
- else
- echo "Error using the script, a jobid is needed! "
- echo "Usage: ${PROG_NAME} Schema -j jobid [-t TableName -s description -d delimiter -e max_error_rows -n name ]"
- echo "Try again! Goodbye!"
- exit 2;
- fi
-fi
-################################################################################
-
-if [ -n "$TABLENAME" ]; then
- bulkloadp.sh -e $MAXERROR -s $Schema -t "$TABLENAME" -j $JOBID -p "$DESC" -d "$DELIMITER" -n "$NAME" -u $USER
-
-else
- bulkloadp.sh -e $MAXERROR -s $Schema -j $JOBID -d "$DELIMITER" -p "$DESC" -n "$NAME" -u $USER
-fi
diff --git a/writeengine/bulk/dbloadp.sh b/writeengine/bulk/dbloadp.sh
deleted file mode 100755
index 550f70a30..000000000
--- a/writeengine/bulk/dbloadp.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-cleanup.sh
-dbbuilder.sh
-bulkloadp.sh
diff --git a/writeengine/bulk/qa-bulkload.py b/writeengine/bulk/qa-bulkload.py
deleted file mode 100644
index ea5436fd8..000000000
--- a/writeengine/bulk/qa-bulkload.py
+++ /dev/null
@@ -1,299 +0,0 @@
-#!/usr/bin/python
-##
-## Bulkloader script by Martin Thomas
-##
-
-import os, sys, glob, shutil, xml.dom.minidom
-import getopt
-import logging
-
-logger = logging.getLogger()
-shdlr = logging.StreamHandler()
-fhdlr = logging.FileHandler(filename='bulkload.log' )
-formatter = logging.Formatter('%(asctime)s:%(levelname)s: %(message)s')
-shdlr.setFormatter(formatter)
-fhdlr.setFormatter(formatter)
-logger.addHandler(shdlr)
-logger.addHandler(fhdlr)
-
-## only report INFO or higher - change to WARNING to silence all logging
-logger.setLevel(logging.INFO)
-
-
-def usage():
- print """
-
- qa-bulkload.py is intended to automate the manual steps required to load the
- database and build indexes from scratch.
-
- - PrimProc will be stopped and started
- - shared memory sgements wil be removed using ipcs-pat
- - database files will be removed
- - dbgen will be run with option 5
- - oid files and job files will be copied to correct locations
- - column data will be parsed and loaded using Job 299
- - index data will be exported, sorted and loaded using Job 300
-
- Options:
- -n or --nocache= : Specify either col or idx and the -c flag will NOT be sent to cpimport
- -u or --usage : Usage message
-
- Example:
- bulkload.py --nocache=idx
- Load the database, do not use cache when building indexes
-
- THIS SPACE LEFT INTENTIONALLY BLANK
- """
-
-def find_paths():
-
- """Find DBRoot and BulkRoot."""
- try:
- config_file = os.environ['COLUMNSTORE_CONFIG_FILE']
- except KeyError:
- try:
- logger.info("Environment variable COLUMNSTORE_CONFIG_FILE not set, looking for system Columnstore.xml")
- config_file = '/usr/local/mariadb/columnstore/etc/Columnstore.xml'
- os.lstat(config_file)
- except:
- logger.error('No config file available')
- sys.exit('No config file available')
- try:
- xmldoc = xml.dom.minidom.parse(config_file)
- bulk_node = xmldoc.getElementsByTagName('BulkRoot')[0]
- db_node = xmldoc.getElementsByTagName('DBRoot')[0]
- bulk_dir = bulk_node.childNodes[0].nodeValue
- data_dir = db_node.childNodes[0].nodeValue
-
- except Exception, e:
- logger.error('Error parsing config file')
- logger.error(e)
- sys.exit('Error parsing config file')
-
- return (bulk_dir, data_dir)
-
-def check_dirs(bulkroot, dbroot):
-
- problem = 0
- res = 0
- reqd_dirs = {
- os.getenv('HOME')+'/genii' : "No genii directory found (contains tools required to continue) (%s)",
- bulkroot: "Bulkroot specified as %s but not found",
- bulkroot+'/job': "No job directory found - needed to store Job xml files (looked in %s)",
- bulkroot+'/data/import': "No data/import directory found - expected %s to hold data to be loaded",
- bulkroot+'/log': "No data/log directory found - expected %s to log into",
- dbroot : "DBroot specified as %s but not found"
- }
- for dir in reqd_dirs.keys():
- try:
- res = os.lstat(dir)
- except:
- problem = 1
- logger.error(reqd_dirs[dir]%dir)
-
- if problem:
- sys.exit(1)
-
-def fix_hwm(job_file):
-
- """Find hwm in xml file and change to 0"""
-
- import re
-
- src_file = open(job_file, 'r')
- dst_file = open(job_file+'.tmp', 'w')
-
- rep = re.compile('hwm="1"')
-
- for line in src_file:
- line = rep.sub('hwm="0"', line)
- dst_file.write(line)
- # use os.rename instead of shutil.move to avoid problems traversing devices
- os.rename(job_file+'.tmp', job_file)
-
-def find_indexes(job_file):
-
- """Find index definitions in job_file and return list of files to sort"""
-
- index_files = []
- try: # try because we may have an old version of python
- xmldoc = xml.dom.minidom.parse(job_file)
-
- for index_node in xmldoc.getElementsByTagName('Index'):
- index_files.append(index_node.getAttribute('mapName'))
- except:
- import re
- f = open(job_file)
- for line in f.read():
- b =re.search('mapName="(CPL_[0-9A-Z_]+)"', line)
- try: # try because not every line will match
- index_files.append(b.group(1))
- except: pass
-
- return index_files
-
-def exec_cmd(cmd, args):
- """Execute command using subprocess module or if that fails,
- use os.system
- """
-
- try:
- import subprocess
-
- try:
- retcode = call(cmd + " "+args, shell=True)
- if retcode < 0:
- print >>sys.stderr, "Child was terminated by signal", -retcode
- sys.exit(-1)
-
- else:
- print >>sys.stderr, "Child returned", retcode
-
- except OSError, e:
-
- print >>sys.stderr, "Execution failed:", e
- sys.exit(-1)
- except:
- logger.info ('Old version of Python - subprocess not available, falling back to os.system')
- logger.info ('Executing: '+cmd+' '+args)
- res = os.system(cmd+' '+args)
- if res:
- logger.error('Bad return code %i from %s'%(res, cmd))
- sys.exit( res )
-
-
-def build_tool(tool):
- """
- Use the tool dictionary to determine if required tool exists
- and build if not
- """
-
- if not os.path.exists(tool['path']+tool['tool']):
- logger.warn ("Building %s before continuing"%tool['tool'])
- curdir=os.getcwd()
- os.chdir(tool['path'])
- exec_cmd(tool['builder'], tool['args'])
- os.chdir(curdir)
-
-def main():
- """
- Bulk load the database..
- Check that we can write OIDfiles, that all required tools exist,
- clean up old files, sort the index inserts and generally rock and roll
- """
- start_dir = curdir=os.getcwd() # remember where we started
-
- if not os.environ.has_key('LD_LIBRARY_PATH'):
- logger.info('No environment variable LD_LIBRARY_PATH')
- else:
- if len(os.getenv('LD_LIBRARY_PATH'))<5:
- logger.info('Suspicous LD_LIBRARY_PATH: %s'%os.getenv('LD_LIBRARY_PATH'))
-
- #-- figure out paths
- home = os.getenv('HOME')
- cache = {}
- cache['idx'] = '-c'
- cache['col'] = '-c'
-
-#-- allow us to specify a write engine branch
- opts, args = getopt.getopt(sys.argv[1:], 'n:u', ['nocache=', 'usage'])
- for opt, arg in opts:
-
- if opt == '-n' or opt == '--nocache':
- if (arg=='idx' or arg=='col'):
- cache[arg] = ''
- logger.info("No cache for %s"% arg)
-
- if opt == '-u' or opt == '--usage':
- usage()
- sys.exit()
-
- (bulkroot, dbroot) = find_paths()
-
- logger.info ("Bulkroot: %s \tDBRoot: %s\n"%(bulkroot, dbroot))
-
- check_dirs(bulkroot, dbroot)
-
- if len(glob.glob(bulkroot+'/data/import/*tbl')) == 0:
- sys.exit("No files for import found in BulkRoot: %s"%(bulkroot))
-
- if len(glob.glob(dbroot+'/000.dir'))==0:
- logger.info("No files found in DBRoot: %s (not fatal)"%dbroot)
-
-## qa version does not build any tools. Cease and desist if any tools missing
-
- toolset = ['dbbuilder', 'cpimport', 'ipcs-pat', 'PrimProc']
- for tool in toolset:
- try:
- res = os.system('which %s'%tool)
- finally:
- if res:
- logger.error("Fatal error: %s not found"%tool)
- sys.exit(-1)
-
-
-
-## clean up before starting
-## remove old db files, removed old temp files, remove shared memory segments,
-## kill old PrimProc and start new one
-
- logger.info ("Removing old DB files")
- exec_cmd('rm -fr ', dbroot+'/000.dir')
-
- logger.info ("Removing old temp files")
- exec_cmd('rm -fr ', bulkroot+'/data/import/*.idx.txt')
-
- logger.info ("Removing shared memory segments")
- exec_cmd('ipcs-pat', '-d')
-
- logger.info("Killing primProc")
- os.system('killall -q -u $USER PrimProc')
-
- logger.info("Starting primProc")
- exec_cmd('PrimProc', "> primproc.log &")
-
-## run dbbuilder
- logger.info ("Building db and indexes (no data inserted)")
- exec_cmd('yes | dbbuilder', ' 5')
-
- logger.info ("Relocating OID files")
- for file in ['colOIDFile.dat', 'dicOIDFile.dat', 'indexOIDFile.dat']:
- # use os.rename instead of shutil.move to avoid problems traversing devices
- os.rename(file, dbroot+'/'+file)
-
- for xmlfile in glob.glob('./Job*xml'):
- logger.info ("Copying %s to %s\n"%(xmlfile, bulkroot+'/job'))
- # use os.rename instead of shutil.move to avoid problems traversing devices
- os.rename(xmlfile, bulkroot+'/job/'+xmlfile)
-
- exec_cmd('time cpimport', '-j 299 -b %s'%cache['col'])
- exec_cmd('time cpimport', '-j 299 -l %s'%cache['col'])
-
- exec_cmd('time cpimport', '-j 300 -i -o %s'%cache['idx'])
-
- logger.info("Over-riding HWM in job file - setting to 0")
- fix_hwm(bulkroot+'/job/Job_300.xml')
-
- ## sort the files after scanning index job file for mapName(s)
- logger.info ("Sorting indexes before insertion")
- indexes = find_indexes(bulkroot+'/job/Job_300.xml')
- for index in indexes:
- data_file='%s/data/import/%s.dat.idx.txt'%(bulkroot, index)
- sort_file ='%s/data/import/%s.dat.idx.sort'%(bulkroot, index)
- exec_cmd('time sort',' -k1 -n %s > %s'%(data_file, sort_file))
- # use os.rename instead of shutil.move to avoid problems traversing devices
- os.rename( sort_file, data_file)
-
- logger.info("Inserting indexes")
- try:
- logger.info("Trying with -m option")
- exec_cmd('cpimport', '-j 300 -m -i -s %s'%cache['idx'])
- except:
- try:
- logger.warn("cpimport with -m option failed, fall back to regular options")
- exec_cmd('cpimport', '-j 300 -i -s %s'%cache['idx'])
- except:
- logger.error("Index load failed")
-
-## the following line allows either interactive use or module import
-if __name__=="__main__": main()