mirror of
https://github.com/sqlite/sqlite.git
synced 2026-10-25 09:59:22 +03:00
92 lines
2.3 KiB
Python
92 lines
2.3 KiB
Python
|
|
import h5py
|
|
import sqlite3
|
|
import numpy as np
|
|
import os
|
|
import sys
|
|
|
|
def sqlite_type_for(dtype):
|
|
# Everything goes in as BLOB; dtype is recorded separately
|
|
return "BLOB"
|
|
|
|
def sanitize(name):
|
|
# HDF5 paths like "/train" → "train"
|
|
return name.strip("/").replace("/", "_")
|
|
|
|
def main(h5_path, sqlite_path):
|
|
if os.path.exists(sqlite_path):
|
|
os.remove(sqlite_path)
|
|
|
|
conn = sqlite3.connect(sqlite_path)
|
|
cur = conn.cursor()
|
|
|
|
with h5py.File(h5_path, "r") as f:
|
|
|
|
dm = "unknown"
|
|
cur.execute("CREATE TABLE meta(field TEXT, value ANY)")
|
|
if "distance" in f.attrs:
|
|
dm = f.attrs["distance"]
|
|
if isinstance(dm, bytes):
|
|
dm = dm.decode("utf-8")
|
|
cur.execute("INSERT INTO meta VALUES('distance', ?)", (dm,))
|
|
|
|
print("VIBE distance metric:", dm)
|
|
|
|
def visit(name, obj):
|
|
if not isinstance(obj, h5py.Dataset):
|
|
return
|
|
|
|
|
|
|
|
data = obj[()]
|
|
table = sanitize(name)
|
|
|
|
print(f"Importing {name} → table '{table}' shape={data.shape} dtype={data.dtype}")
|
|
|
|
# Create table
|
|
cur.execute(f"""
|
|
CREATE TABLE {table} (
|
|
id INTEGER PRIMARY KEY,
|
|
vec BLOB NOT NULL
|
|
)
|
|
""")
|
|
|
|
# Metadata table (created once)
|
|
cur.execute("""
|
|
CREATE TABLE IF NOT EXISTS __meta__ (
|
|
table_name TEXT PRIMARY KEY,
|
|
dtype TEXT,
|
|
shape TEXT
|
|
)
|
|
""")
|
|
|
|
cur.execute(
|
|
"INSERT INTO __meta__ VALUES (?, ?, ?)",
|
|
(table, str(data.dtype), str(data.shape))
|
|
)
|
|
|
|
# Insert rows
|
|
if data.ndim == 1:
|
|
rows = [(i, data[i].tobytes()) for i in range(data.shape[0])]
|
|
else:
|
|
rows = [(i, data[i].tobytes()) for i in range(data.shape[0])]
|
|
|
|
cur.executemany(
|
|
f"INSERT INTO {table} (id, vec) VALUES (?, ?)",
|
|
rows
|
|
)
|
|
|
|
conn.commit()
|
|
|
|
f.visititems(visit)
|
|
|
|
conn.close()
|
|
print("Done.")
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 3:
|
|
print("usage: hdf5_to_sqlite.py input.hdf5 output.sqlite")
|
|
sys.exit(1)
|
|
|
|
main(sys.argv[1], sys.argv[2])
|