[MYSQL] lz4压缩数据结构并使用Python解析

48: literals
15:value for the 4-bits High field
33: (=48-15) remaining length to reach 48

280 literals
15 : value for the 4-bits High field
255 : following byte is maxed, since 280-15 >= 255
10 : (=280 - 15 - 255) remaining length to reach 280

15 literals
15 : value for the 4-bits High field
0 : (=15-15) yes, the zero must be output

def decompress(bdata,decompress_size):
	"""
	input:
		bdata: 压缩数据
		decompress_size : 解压之后的大小
	return: data 解压之后的数据
	不考虑dict和prefix_size了
	"""
	def read_to_less255(tdata,ip):
		length = 0
		while True:
			#t = struct.unpack('<B',aa[ip:ip+1])[0]
			t = tdata[ip]
			ip += 1
			length += t
			if t != 255: # 小于255时,就读完了
				break
		return length,ip
		
	ip = 0 # input pointer  (bdata的指针)
	op = 0 # output pointer (data的指针)
	data = bytearray(decompress_size) # 要返回的数据有这么大
	
	while True:
		token = bdata[ip]
		ip += 1
		ll = token >> 4 # literals length
		if ll == 15:
			tll,ip = read_to_less255(bdata,ip)
			ll += tll
		data[op:op+ll] = bdata[ip:ip+ll] # literals 不可压缩的部分
		op += ll
		ip += ll
		if decompress_size-op < 12:
			if op == decompress_size: # 解压完了, 因为可能没得后面的match部分
				break
			else:
				raise ValueError('Invalid lz4 compress data.')
		#offset = struct.unpack('<H',bdata[ip:ip+2])[0]
		offset = (bdata[ip+1]<<8) | bdata[ip] # 位移真TM好用
		ip += 2
		ml = token & 15 # 后4bit是match length
		if ml == 15:
			tml,ip = read_to_less255(bdata,ip)
			ml += tml
		ml += 4 # 还得加4(minmatch)
		match = op - offset
		data[op:op+ml] = data[match:match+ml] # match实际上是指的原始数据位置的数据,而不是压缩之后的数据位置
		op += ml # 重复的数据只需要移动op就行,ip那没得要移动的.(match部分已经mv了)
	return bytes(data)

create table t20240924_compress_lz4(id int primary key, name varchar(200)) COMPRESSION='lz4';
insert into t20240924_compress_lz4 values(1,'ddcw');
insert into t20240924_compress_lz4 values(2,'https://github.com/ddcw');
insert into t20240924_compress_lz4 values(3,'https://github.com/ddcw/ibd2sql');

# 导入包的代码是重复的, 我就不再写了.
import struct
filename = '/data/mysql_3314/mysqldata/db1/t20240924_compress_lz4.ibd'
f = open(filename,'rb')
f.seek(4*16384,0)
data = f.read(16384)
# 压缩信息
FIL_PAGE_VERSION,FIL_PAGE_ALGORITHM_V1,FIL_PAGE_ORIGINAL_TYPE_V1,FIL_PAGE_ORIGINAL_SIZE_V1,FIL_PAGE_COMPRESS_SIZE_V1 = struct.unpack('>BBHHH',data[26:34])

# lz4解压
data = data[:24] + struct.pack('>H',FIL_PAGE_ORIGINAL_TYPE_V1) + b'\x00'*8 + data[34:38] + lz4.decompress(data[38:38+FIL_PAGE_COMPRESS_SIZE_V1],FIL_PAGE_ORIGINAL_SIZE_V1)

# 验证解压后的数据. 
......

wget https://github.com/ddcw/ibd2sql/archive/refs/heads/main.zip
unzip main.zip
cd ibd2sql-main
python main.py /data/mysql_3314/mysqldata/db1/t20240924_compress_lz4.ibd --sql --ddl