Posted on 2021-11-11 17:27
魔のkyo 阅读(618)
评论(0) 编辑 收藏 引用
import pyarrow as pa
import zlib
class PyArrowEncoder:
def __init__(self, compress_level=-1):
self.compress_level = compress_level
def serialize(self, df):
table = pa.Table.from_pandas(df)
sink = pa.BufferOutputStream()
with pa.ipc.new_stream(sink, table.schema) as writer:
writer.write_table(table)
buf = sink.getvalue()
pybytes = buf.to_pybytes()
return zlib.compress(pybytes, self.compress_level)
def deserialize(self, data):
with pa.ipc.open_stream(zlib.decompress(data)) as reader:
df = reader.read_pandas()
return df