Browse Source

Fix various string/bytes issues with Python 3

tags/nilmdb-2.0.0
Jim Paris 4 years ago
parent
commit
fef3e1d31e
6 changed files with 14 additions and 14 deletions
  1. +2
    -2
      nilmdb/client/numpyclient.py
  2. +5
    -2
      nilmdb/server/bulkdata.py
  3. +1
    -1
      nilmdb/server/nilmdb.py
  4. +4
    -7
      nilmdb/server/rocket.c
  5. +1
    -1
      nilmdb/utils/atomic.py
  6. +1
    -1
      tests/test_client.py

+ 2
- 2
nilmdb/client/numpyclient.py View File

@@ -70,14 +70,14 @@ class NumpyClient(nilmdb.client.client.Client):


# See if we have enough to make the requested Numpy array # See if we have enough to make the requested Numpy array
while total_len >= maxsize: while total_len >= maxsize:
assembled = "".join(chunks)
assembled = b"".join(chunks)
total_len -= maxsize total_len -= maxsize
chunks = [ assembled[maxsize:] ] chunks = [ assembled[maxsize:] ]
block = assembled[:maxsize] block = assembled[:maxsize]
yield to_numpy(block) yield to_numpy(block)


if total_len: if total_len:
yield to_numpy("".join(chunks))
yield to_numpy(b"".join(chunks))


@contextlib.contextmanager @contextlib.contextmanager
def stream_insert_numpy_context(self, path, start = None, end = None, def stream_insert_numpy_context(self, path, start = None, end = None,


+ 5
- 2
nilmdb/server/bulkdata.py View File

@@ -276,7 +276,7 @@ class Table(object):
@classmethod @classmethod
def valid_path(cls, root): def valid_path(cls, root):
"""Return True if a root path is a valid name""" """Return True if a root path is a valid name"""
return "_format" not in root.split(b"/")
return b"_format" not in root.split(b"/")


@classmethod @classmethod
def exists(cls, root): def exists(cls, root):
@@ -538,7 +538,10 @@ class Table(object):
ret.append(f.extract_string(offset, count)) ret.append(f.extract_string(offset, count))
remaining -= count remaining -= count
row += count row += count
return "".join(ret)
if binary:
return b"".join(ret)
else:
return "".join(ret)


def __getitem__(self, row): def __getitem__(self, row):
"""Extract timestamps from a row, with table[n] notation.""" """Extract timestamps from a row, with table[n] notation."""


+ 1
- 1
nilmdb/server/nilmdb.py View File

@@ -638,7 +638,7 @@ class NilmDB(object):


if count: if count:
return matched return matched
return ("".join(result), restart)
return (b"".join(result), restart)


def stream_remove(self, path, start = None, end = None): def stream_remove(self, path, start = None, end = None):
""" """


+ 4
- 7
nilmdb/server/rocket.c View File

@@ -279,10 +279,7 @@ static PyObject *Rocket_append_string(Rocket *self, PyObject *args)
union64_t t64; union64_t t64;
int i; int i;


/* It would be nice to use 't#' instead of 's' for data,
but we need the null termination for strto*. If we had
strnto* that took a length, we could use t# and not require
a copy. */
/* Input data is Unicode */
if (!PyArg_ParseTuple(args, "isiiLLL:append_string", &count, if (!PyArg_ParseTuple(args, "isiiLLL:append_string", &count,
&data, &offset, &linenum, &data, &offset, &linenum,
&ll1, &ll2, &ll3)) &ll1, &ll2, &ll3))
@@ -443,7 +440,7 @@ static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
timestamp_t end; timestamp_t end;
timestamp_t last_timestamp; timestamp_t last_timestamp;


if (!PyArg_ParseTuple(args, "it#iiLLL:append_binary",
if (!PyArg_ParseTuple(args, "iy#iiLLL:append_binary",
&count, &data, &data_len, &offset, &count, &data, &data_len, &offset,
&linenum, &ll1, &ll2, &ll3)) &linenum, &ll1, &ll2, &ll3))
return NULL; return NULL;
@@ -493,7 +490,7 @@ static PyObject *Rocket_append_binary(Rocket *self, PyObject *args)
} }


/**** /****
* Extract to string
* Extract to a Unicode string
*/ */


static PyObject *Rocket_extract_string(Rocket *self, PyObject *args) static PyObject *Rocket_extract_string(Rocket *self, PyObject *args)
@@ -601,7 +598,7 @@ err:
} }


/**** /****
* Extract to binary string containing raw little-endian binary data
* Extract to binary bytes object containing raw little-endian binary data
*/ */
static PyObject *Rocket_extract_binary(Rocket *self, PyObject *args) static PyObject *Rocket_extract_binary(Rocket *self, PyObject *args)
{ {


+ 1
- 1
nilmdb/utils/atomic.py View File

@@ -7,7 +7,7 @@ def replace_file(filename, content):
given contents. This is intended to be 'pretty good on most given contents. This is intended to be 'pretty good on most
OSes', but not necessarily bulletproof.""" OSes', but not necessarily bulletproof."""


newfilename = filename + ".new"
newfilename = filename + b".new"


# Write to new file, flush it # Write to new file, flush it
with open(newfilename, "wb") as f: with open(newfilename, "wb") as f:


+ 1
- 1
tests/test_client.py View File

@@ -333,7 +333,7 @@ class TestClient(object):
with assert_raises(ClientError) as e: with assert_raises(ClientError) as e:
list(client.stream_extract("/newton/prep", list(client.stream_extract("/newton/prep",
count = True, binary = True)) count = True, binary = True))
data = "".join(client.stream_extract("/newton/prep", binary = True))
data = b"".join(client.stream_extract("/newton/prep", binary = True))
# Quick check using struct # Quick check using struct
unpacker = struct.Struct("<qffffffff") unpacker = struct.Struct("<qffffffff")
out = [] out = []


Loading…
Cancel
Save