484 lines
18 KiB
Python
484 lines
18 KiB
Python
import os
|
|
import sys
|
|
import zlib
|
|
import nbt
|
|
import random
|
|
import time
|
|
import logging
|
|
from io import BytesIO
|
|
import sqlite3
|
|
from serialize import *
|
|
from itemstack import *
|
|
from tile_entities import te_convert
|
|
from entities import e_convert
|
|
|
|
logger = logging.getLogger('block')
|
|
|
|
class MCMap:
|
|
"""A MC map"""
|
|
def __init__(self, path):
|
|
self.world_path = os.path.join(path, "region")
|
|
self.chunk_pos = []
|
|
for ext in ["mca", "mcr"]:
|
|
filenames = [i for i in os.listdir(self.world_path)
|
|
if i.endswith("."+ext)]
|
|
if len(filenames) > 0:
|
|
self.ext = ext
|
|
break
|
|
chunkCounta = 0
|
|
chunkCountb = 0
|
|
for filename in filenames:
|
|
chunkCounta += 1
|
|
s = filename.split(".")
|
|
cx, cz = int(s[1])*32, int(s[2])*32
|
|
|
|
with open(os.path.join(self.world_path, filename), "rb") as f:
|
|
for chkx in range(cx, cx+32):
|
|
for chkz in range(cz, cz+32):
|
|
offset = ((chkx%32) + 32*(chkz%32))*4
|
|
f.seek(offset)
|
|
if bytesToInt(f.read(3)) != 0:
|
|
self.chunk_pos.append((chkx, chkz))
|
|
chunkCountb += 1
|
|
|
|
def getChunk(self, chkx, chkz):
|
|
return MCChunk(chkx, chkz, self.world_path, self.ext)
|
|
|
|
def getBlocksIterator(self):
|
|
num_chunks = len(self.chunk_pos)
|
|
chunk_ix = 0
|
|
t0 = time.time()
|
|
for chkx, chkz in self.chunk_pos:
|
|
if chunk_ix%10 == 0:
|
|
if chunk_ix > 0:
|
|
td = time.time() - t0 # wall clock time spent
|
|
tr = ((num_chunks * td) / chunk_ix) - td # time remaining
|
|
eta = "{:d}:{:02d}:{:02d}".format(int(tr/3600), int(tr/60) %60, int(tr) % 60)
|
|
else:
|
|
eta = "??:??:??"
|
|
print('Processed %d / %d chunks, ETA %s h:m:s' % (chunk_ix, num_chunks, eta), end='\r')
|
|
sys.stdout.flush()
|
|
chunk_ix += 1
|
|
blocks = self.getChunk(chkx, chkz).blocks
|
|
for block in blocks:
|
|
yield block
|
|
print()
|
|
|
|
class MCChunk:
|
|
"""A 16x16 column of nodes"""
|
|
def __init__(self, chkx, chkz, path, ext):
|
|
filename = os.path.join(path,
|
|
"r.{}.{}.{}".format(chkx//32, chkz//32, ext))
|
|
with open(filename, "rb") as f:
|
|
ofs = ((chkx%32) + 32*(chkz%32))*4
|
|
f.seek(ofs)
|
|
offset = bytesToInt(f.read(3)) << 12
|
|
f.seek(offset)
|
|
length = bytesToInt(f.read(4))
|
|
compression_type = bytesToInt(f.read(1))
|
|
data = f.read(length - 1) # 1 byte for compression_type
|
|
if compression_type == 1: # Gzip
|
|
udata = zlib.decompress(data, 15+16)
|
|
elif compression_type == 2: # Zlib
|
|
udata = zlib.decompress(data)
|
|
else:
|
|
raise ValueError("Unsupported compression type")
|
|
raw_data = nbt.read(udata)['']['Level']
|
|
self.blocks = []
|
|
if ext == "mca":
|
|
# Anvil file format
|
|
for section in raw_data["Sections"]:
|
|
self.blocks.append(MCBlock(raw_data, (chkx, chkz), section["Y"], True))
|
|
else:
|
|
for yslice in range(8):
|
|
self.blocks.append(MCBlock(raw_data, (chkx, chkz), yslice, False))
|
|
|
|
|
|
|
|
class MCBlock:
|
|
"""A 16x16x16 block"""
|
|
def __init__(self, chunk, chunkpos, yslice, is_anvil=True):
|
|
if is_anvil:
|
|
# the x axis has to be inverted to convert to minetest (the chunk location is at the L lower corner, so subtract one or there would be 2 chunks at 0).
|
|
# This converts the chunk location (node level data is converted by reverse_X_axis)
|
|
self.pos = (-chunkpos[0]-1, yslice, chunkpos[1])
|
|
# Find the slice
|
|
for section in chunk["Sections"]:
|
|
if section["Y"] == yslice:
|
|
self.from_section(section)
|
|
break
|
|
else:
|
|
self.pos = (chunkpos[0], yslice, chunkpos[1])
|
|
# No luck, we have to convert
|
|
self.from_chunk(chunk, yslice)
|
|
|
|
self.tile_entities = []
|
|
for te in chunk["TileEntities"]:
|
|
if (te["y"]>>4) == yslice:
|
|
t = te.copy()
|
|
# Entity data stores it's own position information, so has to be modified independently in addition to other blocks.
|
|
t["y"] &= 0xf
|
|
t["y"] = t["y"] -16
|
|
# within the chunk x position has to be inverted to convert to minetest:-
|
|
if is_anvil:
|
|
t["x"] = self.pos[0]*16 + 15-t["x"]%16
|
|
self.tile_entities.append(t)
|
|
|
|
self.entities = []
|
|
for e in chunk["Entities"]:
|
|
t = e.copy()
|
|
self.entities.append(t)
|
|
|
|
|
|
@staticmethod
|
|
def expand_half_bytes(l):
|
|
# This function reverses x axis node order within each slice, and
|
|
# expands the 4bit sequences into 8bit sequences
|
|
|
|
l3=[]
|
|
for y in range(0,2047,128):
|
|
for z in range(120,-1,-8):
|
|
#for z in range(0,127,8):
|
|
locSt=y+z
|
|
l2 = l[locSt:locSt+8]
|
|
#for i in reversed(l2):
|
|
for i in l2:
|
|
l3.append(i&0xf)
|
|
l3.append((i>>4)&0xf)
|
|
return l3
|
|
|
|
|
|
@staticmethod
|
|
def reverse_X_axis(l):
|
|
# Anvil format is YZX ((y * 16 + z) * 16 + x)
|
|
# block data is actually u12 per data point (ie per node)
|
|
# but is split into u8 (='blocks') dealt with in reverse_X_axis() and u4 (='data') dealt with in expand_half_bytes()
|
|
# NB data, skylight and blocklight are only 4bits of data
|
|
|
|
# To convert minecraft to minetest coordinates you must invert the x order while leaving y and z the same
|
|
# 2017/02/14 : In order to have north on the good side, we'll rather invert Z axis
|
|
l3=[]
|
|
for y in range(0,4095,256):
|
|
#for z in range(0,255,16):
|
|
for z in range(240,-1,-16):
|
|
locSt=y+z
|
|
l2 = l[locSt:locSt+16]
|
|
#for i in reversed(l2):
|
|
for i in l2:
|
|
l3.append(i)
|
|
return l3
|
|
|
|
|
|
|
|
def from_section(self, section):
|
|
self.blocks = self.reverse_X_axis(section["Blocks"])
|
|
self.data = self.expand_half_bytes(section["Data"])
|
|
self.sky_light = self.expand_half_bytes(section["SkyLight"])
|
|
self.block_light = self.expand_half_bytes(section["BlockLight"])
|
|
|
|
@staticmethod
|
|
def extract_slice(data, yslice):
|
|
data2 = [0]*4096
|
|
k = yslice << 4
|
|
k2 = 0
|
|
# Beware: impossible to understand code
|
|
# Sorry, but it has to be as fast as possible,
|
|
# as it is one bottleneck
|
|
# Basically: order is changed from XZY to YZX
|
|
for y in range(16):
|
|
for z in range(16):
|
|
for x in range(16):
|
|
data2[k2] = data[k]
|
|
k2 += 1
|
|
k += 2048
|
|
k = (k&0x7ff)+128
|
|
k = (k&0x7f)+1
|
|
return data2
|
|
|
|
@staticmethod
|
|
def extract_slice_half_bytes(data, yslice):
|
|
data2 = [0]*4096
|
|
k = yslice << 3
|
|
k2 = 0
|
|
k3 = 256 # One layer above the previous one
|
|
# Beware: impossible to understand code
|
|
# Even worse than before: that time we've got
|
|
# to extract half bytes at the same time
|
|
# Again, order is changed from XZY to YZX
|
|
for y in range(0, 16, 2): # 2 values for y at a time
|
|
for z in range(16):
|
|
for x in range(16):
|
|
data2[k2] = data[k]&0xf
|
|
data2[k3] = (data[k]>>4)&0xf
|
|
k2 += 1
|
|
k3 += 1
|
|
k += 1024
|
|
k = (k&0x3ff)+64
|
|
k = (k&0x3f)+1
|
|
k2 += 256 # Skip a layer
|
|
k3 += 256
|
|
return data2
|
|
|
|
def from_chunk(self, chunk, yslice):
|
|
self.blocks = self.extract_slice(chunk["Blocks"], yslice)
|
|
self.data = self.extract_slice_half_bytes(chunk["Data"], yslice)
|
|
self.sky_light = self.extract_slice_half_bytes(chunk["SkyLight"], yslice)
|
|
self.block_light = self.extract_slice_half_bytes(chunk["BlockLight"], yslice)
|
|
|
|
class MTBlock:
|
|
def __init__(self, name_id_mapping):
|
|
self.name_id_mapping = name_id_mapping
|
|
self.content = [0]*4096
|
|
self.mcblockidentifier = ['']*4096
|
|
self.param1 = [0]*4096
|
|
self.param2 = [0]*4096
|
|
self.metadata = {}
|
|
self.pos = (0, 0, 0)
|
|
|
|
def fromMCBlock(self, mcblock, conversion_table):
|
|
logger.debug('***fromMCBlock: Starting New Block***')
|
|
|
|
self.timers = []
|
|
self.pos = (mcblock.pos[0], mcblock.pos[1]-4, mcblock.pos[2])
|
|
content = self.content
|
|
mcblockidentifier = self.mcblockidentifier
|
|
param1 = self.param1
|
|
param2 = self.param2
|
|
blocks = mcblock.blocks
|
|
data = mcblock.data
|
|
skylight = mcblock.sky_light
|
|
blocklight = mcblock.block_light
|
|
|
|
# now load all the nodes in the 16x16x16 (=4096) block
|
|
for i in range(4096):
|
|
content[i], param2[i] = conversion_table[blocks[i]][data[i]]
|
|
param1[i] = max(blocklight[i], skylight[i])|(blocklight[i]<<4)
|
|
mcblockidentifier[i] = str(blocks[i]) + ':' + str(data[i])
|
|
|
|
def isdoor(b):
|
|
return b == 64 or b == 71 or (b >= 193 and b <= 197)
|
|
|
|
# water
|
|
if (blocks[i] == 9 or blocks[i] == 11) and (data[i] == 0):
|
|
content[i], param2[i] = conversion_table[blocks[i]][data[i]]
|
|
elif blocks[i] >= 8 and blocks[i] <= 11:
|
|
# nop, exit case
|
|
pass
|
|
# pressure plates - append mesecons node timer
|
|
elif blocks[i] == 70 or blocks[i] == 72:
|
|
self.timers.append(((i&0xf)|((i>>4)&0xf)<<8|((i>>8)&0xf)<<4, 100, 0))
|
|
# rotate lily pads randomly
|
|
elif blocks[i] == 111:
|
|
param2[i] = random.randint(0,3)
|
|
# melon/pumpkin blocks
|
|
elif blocks[i] == 86 or blocks[i] == 103:
|
|
param2[i] = random.randint(0,23)
|
|
# grass of varying length randomly
|
|
elif blocks[i] == 31 and data[i] == 1:
|
|
content[i], param2[i] = conversion_table[931][random.randint(0,4)]
|
|
# fix doors based on top/bottom bits
|
|
elif isdoor(blocks[i]) and data[i] < 8: # bottom part
|
|
above = i + 256
|
|
if (above >= 4096):
|
|
logger.warning('Unable to fix door - top part is across block boundary! (%d >= 4096)' % above)
|
|
elif isdoor(blocks[above]) and data[above] < 8:
|
|
logger.warning('Unable to fix door - bottom part 0x%x on top of bottom part 0x%x!', data[i], data[above])
|
|
else:
|
|
d_right = data[above] & 1 # 0 - left, 1 - right
|
|
d_open = data[i] & 4 # 0 - closed, 1 - open
|
|
d_face = data[i] & 3 # n,e,s,w orientation
|
|
alt = 964
|
|
if blocks[i] == 71:
|
|
alt = 966
|
|
content[i], param2[i] = conversion_table[alt][d_face|d_open|(d_right<<3)]
|
|
if d_right == 1:
|
|
self.metadata[(i & 0xf, (i>>8) & 0xf, (i>>4) & 0xf)] = ({ "right": "1" }, {})
|
|
|
|
elif content[i]==0 and param2[i]==0 and not (blocks[i]==0):
|
|
logger.warning('Unknown Minecraft Block:' + str(mcblockidentifier[i])) # This is the minecraft ID#/data as listed in map_content.txt
|
|
|
|
for te in mcblock.tile_entities:
|
|
id = te["id"]
|
|
x, y, z = -te["x"] - 1, te["y"], -te["z"] - 1
|
|
index = ((y&0xf)<<8)|((z&0xf)<<4)|(x&0xf)
|
|
f = te_convert.get(id.lower(), lambda arg: (None, None, None)) # Do nothing if not found
|
|
block, p2, meta = f(te)
|
|
logger.debug('EntityInfoPre: ' +str(te))
|
|
logger.debug('EntityInfoPost: ' +' y='+str(y)+' z='+str(z)+' x='+str(x)+' Meta:'+str(meta))
|
|
# NB block and p2 never seems to be returned, but if this is important, then just change the above 'meta' to 'f(te)'
|
|
|
|
if block != None:
|
|
blocks[index] = block
|
|
if p2 != None:
|
|
param2[index] = p2
|
|
if meta != None:
|
|
try:
|
|
p = meta[0]["_plant"]
|
|
if p > 15:
|
|
content[index], param2[index] = conversion_table[941][p&0xf]
|
|
else:
|
|
content[index], param2[index] = conversion_table[940][p]
|
|
except:
|
|
self.metadata[(x&0xf, y&0xf, z&0xf)] = meta
|
|
|
|
for e in mcblock.entities:
|
|
id = e["id"]
|
|
f = e_convert.get(id.lower(), lambda arg: (None, None, None)) # Do nothing if not found
|
|
block, p2, meta = f(e)
|
|
|
|
def save(self):
|
|
os = BytesIO()
|
|
writeU8(os, 25) # Version
|
|
|
|
#flags
|
|
flags = 0x00
|
|
if self.pos[1] < -1:
|
|
flags |= 0x01 #is_underground
|
|
flags |= 0x02 #day_night_differs
|
|
flags |= 0x04 #lighting_expired
|
|
flags |= 0x08 #generated
|
|
writeU8(os, flags)
|
|
|
|
writeU8(os, 2) # content_width
|
|
writeU8(os, 2) # params_width
|
|
|
|
cbuffer = BytesIO()
|
|
# Bulk node data
|
|
content = self.content
|
|
k = 0
|
|
nimap = {}
|
|
rev_nimap = []
|
|
first_free_content = 0
|
|
for z in range(16):
|
|
for y in range(16):
|
|
for x in range(16):
|
|
#writeU16(cbuffer, content[k])
|
|
c = content[k]
|
|
if c in nimap:
|
|
writeU16(cbuffer, nimap[c])
|
|
else:
|
|
nimap[c] = first_free_content
|
|
writeU16(cbuffer, first_free_content)
|
|
rev_nimap.append(c)
|
|
first_free_content += 1
|
|
k += 1
|
|
k += (256-16)
|
|
k += (16-16*256)
|
|
param1 = self.param1
|
|
k = 0
|
|
for z in range(16):
|
|
for y in range(16):
|
|
for x in range(16):
|
|
writeU8(cbuffer, param1[k])
|
|
k += 1
|
|
k += (256-16)
|
|
k += (16-16*256)
|
|
param2 = self.param2
|
|
k = 0
|
|
for z in range(16):
|
|
for y in range(16):
|
|
for x in range(16):
|
|
writeU8(cbuffer, param2[k])
|
|
k += 1
|
|
k += (256-16)
|
|
k += (16-16*256)
|
|
os.write(zlib.compress(cbuffer.getvalue()))
|
|
|
|
# Nodemeta
|
|
meta = self.metadata
|
|
|
|
cbuffer = BytesIO()
|
|
writeU8(cbuffer, 1) # Version
|
|
writeU16(cbuffer, len(meta))
|
|
for pos, data in meta.items():
|
|
writeU16(cbuffer, (pos[2]<<8)|(pos[1]<<4)|pos[0])
|
|
writeU32(cbuffer, len(data[0]))
|
|
for name, val in data[0].items():
|
|
writeString(cbuffer, name)
|
|
writeLongString(cbuffer, str(val))
|
|
serialize_inv(cbuffer, data[1])
|
|
os.write(zlib.compress(cbuffer.getvalue()))
|
|
|
|
# Static objects
|
|
writeU8(os, 0) # Version
|
|
writeU16(os, 0) # Number of objects
|
|
|
|
# Timestamp
|
|
writeU32(os, 0xffffffff) # BLOCK_TIMESTAMP_UNDEFINED
|
|
|
|
# Name-ID mapping
|
|
writeU8(os, 0) # Version
|
|
writeU16(os, len(rev_nimap))
|
|
for i in range(len(rev_nimap)):
|
|
writeU16(os, i)
|
|
writeString(os, self.name_id_mapping[rev_nimap[i]])
|
|
|
|
# Node timer
|
|
writeU8(os, 2+4+4) # Timer data len
|
|
writeU16(os, len(self.timers)) # Number of timers
|
|
if len(self.timers) > 0:
|
|
logger.info('wrote ' + str(len(self.timers)) + ' node timers')
|
|
for i in range(len(self.timers)):
|
|
writeU16(os, self.timers[i][0])
|
|
writeU32(os, self.timers[i][1])
|
|
writeU32(os, self.timers[i][2])
|
|
|
|
return os.getvalue()
|
|
|
|
class MTMap:
|
|
def __init__(self, path):
|
|
self.world_path = path
|
|
self.blocks = []
|
|
|
|
@staticmethod
|
|
def getBlockAsInteger(p):
|
|
return p[0]+4096*(p[1]+4096*p[2])
|
|
|
|
@staticmethod
|
|
def fromMCMapBlocksIterator(mcmap, name_id_mapping, conversion_table):
|
|
for mcblock in mcmap.getBlocksIterator():
|
|
mtblock = MTBlock(name_id_mapping)
|
|
mtblock.fromMCBlock(mcblock, conversion_table)
|
|
yield mtblock
|
|
|
|
def fromMCMap(self, mcmap, nimap, ct):
|
|
self.blocks = self.fromMCMapBlocksIterator(mcmap, nimap, ct)
|
|
|
|
def save(self):
|
|
conn = sqlite3.connect(os.path.join(self.world_path, "map.sqlite"))
|
|
cur = conn.cursor()
|
|
|
|
cur.execute("CREATE TABLE IF NOT EXISTS `blocks` (\
|
|
`pos` INT NOT NULL PRIMARY KEY, `data` BLOB);")
|
|
|
|
num_saved = 0
|
|
for block in self.blocks:
|
|
if num_saved%100 == 0:
|
|
#print("Saved", num_saved, "blocks")
|
|
conn.commit()
|
|
num_saved += 1
|
|
cur.execute("INSERT INTO blocks VALUES (?,?)",
|
|
# (self.getBlockAsInteger((-block.pos[0],block.pos[1],block.pos[2])),
|
|
(self.getBlockAsInteger((-block.pos[0],block.pos[1],-block.pos[2])),
|
|
block.save()))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# Tests
|
|
from random import randrange
|
|
t = [randrange(256) for i in range(2048*8)]
|
|
assert(MCBlock.extract_slice(MCBlock.expand_half_bytes(t), 0)
|
|
== MCBlock.extract_slice_half_bytes(t, 0))
|
|
|
|
from time import time
|
|
t0 = time()
|
|
s1 = MCBlock.extract_slice(MCBlock.expand_half_bytes(t), 1)
|
|
print(time()-t0)
|
|
t0 = time()
|
|
s2 = MCBlock.extract_slice_half_bytes(t, 1)
|
|
print(time()-t0)
|
|
assert(s1 == s2)
|