#!/usr/bin/python from printf import printf import time import re import numpy as np import itertools import struct import array class Timer(): def __init__(self, arg): self.arg = arg def __enter__(self): self.start = time.time() def __exit__(self, *args): printf("%s: %f klines/sec\n", self.arg, 1e3 / (time.time() - self.start)) def read_ascii(): for n in xrange(1000000): yield (1234, 2345, 3456, 4576, 5678, 6789) # for n, line in enumerate(open('1m.raw', 'r')): # out = [0]*6 # tmp = [ int(i,10) for i in line.partition('#')[0].split() ] # out[0:len(tmp)] = tmp # if (n % 100000 == 0): # printf("line %d = %s\n", n, str(out)) # yield out def test_struct1(): """write with struct.pack""" f = open('1m.bin', 'wb') for out in read_ascii(): s = struct.pack('!HHHHHH', *out) f.write(s) def test_struct2(): """use constant format string""" f = open('1m.bin', 'wb') packer = struct.Struct('!HHHHHH') for out in read_ascii(): f.write(packer.pack(*out)) f.close() printf("size was %d\n", packer.size) def test_struct3(): """like struct1, with timestamp""" f = open('1m.bin', 'wb') for out in read_ascii(): s = struct.pack('!dHHHHHH', time.time(), *out) f.write(s) def test_struct4(): """like struct2, with timestamp""" f = open('1m.bin', 'wb') packer = struct.Struct('!dHHHHHH') for out in read_ascii(): f.write(packer.pack(time.time(), *out)) f.close() printf("size was %d\n", packer.size) #raise Exception('done') with Timer("struct1"): test_struct1() # 1089k with Timer("struct2"): test_struct2() # 1249k with Timer("struct3"): test_struct3() # 845k with Timer("struct4"): test_struct4() # 922k # This seems fast enough for writing new data, since it's faster than # we read ascii data anyway. Use e.g. struct4