import os import sys import zlib import nbt import random import time import logging from io import BytesIO import sqlite3 from serialize import * from itemstack import * from tile_entities import te_convert from entities import e_convert logger = logging.getLogger('block') class MCMap: """A MC map""" def __init__(self, path): self.world_path = os.path.join(path, "region") self.chunk_pos = [] for ext in ["mca", "mcr"]: filenames = [i for i in os.listdir(self.world_path) if i.endswith("."+ext)] if len(filenames) > 0: self.ext = ext break chunkCounta = 0 chunkCountb = 0 for filename in filenames: chunkCounta += 1 s = filename.split(".") cx, cz = int(s[1])*32, int(s[2])*32 with open(os.path.join(self.world_path, filename), "rb") as f: for chkx in range(cx, cx+32): for chkz in range(cz, cz+32): offset = ((chkx%32) + 32*(chkz%32))*4 f.seek(offset) if bytesToInt(f.read(3)) != 0: self.chunk_pos.append((chkx, chkz)) chunkCountb += 1 def getChunk(self, chkx, chkz): return MCChunk(chkx, chkz, self.world_path, self.ext) def getBlocksIterator(self): num_chunks = len(self.chunk_pos) chunk_ix = 0 t0 = time.time() for chkx, chkz in self.chunk_pos: if chunk_ix%10 == 0: if chunk_ix > 0: td = time.time() - t0 # wall clock time spent tr = ((num_chunks * td) / chunk_ix) - td # time remaining eta = "{:d}:{:02d}:{:02d}".format(int(tr/3600), int(tr/60) %60, int(tr) % 60) else: eta = "??:??:??" print('Processed %d / %d chunks, ETA %s h:m:s' % (chunk_ix, num_chunks, eta), end='\r') sys.stdout.flush() chunk_ix += 1 blocks = self.getChunk(chkx, chkz).blocks for block in blocks: yield block print() class MCChunk: """A 16x16 column of nodes""" def __init__(self, chkx, chkz, path, ext): filename = os.path.join(path, "r.{}.{}.{}".format(chkx//32, chkz//32, ext)) with open(filename, "rb") as f: ofs = ((chkx%32) + 32*(chkz%32))*4 f.seek(ofs) offset = bytesToInt(f.read(3)) << 12 f.seek(offset) length = bytesToInt(f.read(4)) compression_type = bytesToInt(f.read(1)) data = f.read(length - 1) # 1 byte for compression_type if compression_type == 1: # Gzip udata = zlib.decompress(data, 15+16) elif compression_type == 2: # Zlib udata = zlib.decompress(data) else: raise ValueError("Unsupported compression type") raw_data = nbt.read(udata)['']['Level'] self.blocks = [] if ext == "mca": # Anvil file format for section in raw_data["Sections"]: self.blocks.append(MCBlock(raw_data, (chkx, chkz), section["Y"], True)) else: for yslice in range(8): self.blocks.append(MCBlock(raw_data, (chkx, chkz), yslice, False)) class MCBlock: """A 16x16x16 block""" def __init__(self, chunk, chunkpos, yslice, is_anvil=True): if is_anvil: # the x axis has to be inverted to convert to minetest (the chunk location is at the L lower corner, so subtract one or there would be 2 chunks at 0). # This converts the chunk location (node level data is converted by reverse_X_axis) self.pos = (-chunkpos[0]-1, yslice, chunkpos[1]) # Find the slice for section in chunk["Sections"]: if section["Y"] == yslice: self.from_section(section) break else: self.pos = (chunkpos[0], yslice, chunkpos[1]) # No luck, we have to convert self.from_chunk(chunk, yslice) self.tile_entities = [] for te in chunk["TileEntities"]: if (te["y"]>>4) == yslice: t = te.copy() # Entity data stores it's own position information, so has to be modified independently in addition to other blocks. t["y"] &= 0xf t["y"] = t["y"] -16 # within the chunk x position has to be inverted to convert to minetest:- if is_anvil: t["x"] = self.pos[0]*16 + 15-t["x"]%16 self.tile_entities.append(t) self.entities = [] for e in chunk["Entities"]: t = e.copy() self.entities.append(t) @staticmethod def expand_half_bytes(l): # This function reverses x axis node order within each slice, and # expands the 4bit sequences into 8bit sequences l3=[] for y in range(0,2047,128): for z in range(120,-1,-8): #for z in range(0,127,8): locSt=y+z l2 = l[locSt:locSt+8] #for i in reversed(l2): for i in l2: l3.append(i&0xf) l3.append((i>>4)&0xf) return l3 @staticmethod def reverse_X_axis(l): # Anvil format is YZX ((y * 16 + z) * 16 + x) # block data is actually u12 per data point (ie per node) # but is split into u8 (='blocks') dealt with in reverse_X_axis() and u4 (='data') dealt with in expand_half_bytes() # NB data, skylight and blocklight are only 4bits of data # To convert minecraft to minetest coordinates you must invert the x order while leaving y and z the same # 2017/02/14 : In order to have north on the good side, we'll rather invert Z axis l3=[] for y in range(0,4095,256): #for z in range(0,255,16): for z in range(240,-1,-16): locSt=y+z l2 = l[locSt:locSt+16] #for i in reversed(l2): for i in l2: l3.append(i) return l3 def from_section(self, section): self.blocks = self.reverse_X_axis(section["Blocks"]) self.data = self.expand_half_bytes(section["Data"]) self.sky_light = self.expand_half_bytes(section["SkyLight"]) self.block_light = self.expand_half_bytes(section["BlockLight"]) @staticmethod def extract_slice(data, yslice): data2 = [0]*4096 k = yslice << 4 k2 = 0 # Beware: impossible to understand code # Sorry, but it has to be as fast as possible, # as it is one bottleneck # Basically: order is changed from XZY to YZX for y in range(16): for z in range(16): for x in range(16): data2[k2] = data[k] k2 += 1 k += 2048 k = (k&0x7ff)+128 k = (k&0x7f)+1 return data2 @staticmethod def extract_slice_half_bytes(data, yslice): data2 = [0]*4096 k = yslice << 3 k2 = 0 k3 = 256 # One layer above the previous one # Beware: impossible to understand code # Even worse than before: that time we've got # to extract half bytes at the same time # Again, order is changed from XZY to YZX for y in range(0, 16, 2): # 2 values for y at a time for z in range(16): for x in range(16): data2[k2] = data[k]&0xf data2[k3] = (data[k]>>4)&0xf k2 += 1 k3 += 1 k += 1024 k = (k&0x3ff)+64 k = (k&0x3f)+1 k2 += 256 # Skip a layer k3 += 256 return data2 def from_chunk(self, chunk, yslice): self.blocks = self.extract_slice(chunk["Blocks"], yslice) self.data = self.extract_slice_half_bytes(chunk["Data"], yslice) self.sky_light = self.extract_slice_half_bytes(chunk["SkyLight"], yslice) self.block_light = self.extract_slice_half_bytes(chunk["BlockLight"], yslice) class MTBlock: def __init__(self, name_id_mapping): self.name_id_mapping = name_id_mapping self.content = [0]*4096 self.mcblockidentifier = ['']*4096 self.param1 = [0]*4096 self.param2 = [0]*4096 self.metadata = {} self.pos = (0, 0, 0) def fromMCBlock(self, mcblock, conversion_table): logger.debug('***fromMCBlock: Starting New Block***') self.timers = [] self.pos = (mcblock.pos[0], mcblock.pos[1]-4, mcblock.pos[2]) content = self.content mcblockidentifier = self.mcblockidentifier param1 = self.param1 param2 = self.param2 blocks = mcblock.blocks data = mcblock.data skylight = mcblock.sky_light blocklight = mcblock.block_light # now load all the nodes in the 16x16x16 (=4096) block for i in range(4096): content[i], param2[i] = conversion_table[blocks[i]][data[i]] param1[i] = max(blocklight[i], skylight[i])|(blocklight[i]<<4) mcblockidentifier[i] = str(blocks[i]) + ':' + str(data[i]) def isdoor(b): return b == 64 or b == 71 or (b >= 193 and b <= 197) # water if (blocks[i] == 9 or blocks[i] == 11) and (data[i] == 0): content[i], param2[i] = conversion_table[blocks[i]][data[i]] elif blocks[i] >= 8 and blocks[i] <= 11: # nop, exit case pass # pressure plates - append mesecons node timer elif blocks[i] == 70 or blocks[i] == 72: self.timers.append(((i&0xf)|((i>>4)&0xf)<<8|((i>>8)&0xf)<<4, 100, 0)) # rotate lily pads randomly elif blocks[i] == 111: param2[i] = random.randint(0,3) # melon/pumpkin blocks elif blocks[i] == 86 or blocks[i] == 103: param2[i] = random.randint(0,23) # grass of varying length randomly elif blocks[i] == 31 and data[i] == 1: content[i], param2[i] = conversion_table[931][random.randint(0,4)] # fix doors based on top/bottom bits elif isdoor(blocks[i]) and data[i] < 8: # bottom part above = i + 256 if (above >= 4096): logger.warning('Unable to fix door - top part is across block boundary! (%d >= 4096)' % above) elif isdoor(blocks[above]) and data[above] < 8: logger.warning('Unable to fix door - bottom part 0x%x on top of bottom part 0x%x!', data[i], data[above]) else: d_right = data[above] & 1 # 0 - left, 1 - right d_open = data[i] & 4 # 0 - closed, 1 - open d_face = data[i] & 3 # n,e,s,w orientation alt = 964 if blocks[i] == 71: alt = 966 content[i], param2[i] = conversion_table[alt][d_face|d_open|(d_right<<3)] if d_right == 1: self.metadata[(i & 0xf, (i>>8) & 0xf, (i>>4) & 0xf)] = ({ "right": "1" }, {}) elif content[i]==0 and param2[i]==0 and not (blocks[i]==0): logger.warning('Unknown Minecraft Block:' + str(mcblockidentifier[i])) # This is the minecraft ID#/data as listed in map_content.txt for te in mcblock.tile_entities: id = te["id"] x, y, z = -te["x"] - 1, te["y"], -te["z"] - 1 index = ((y&0xf)<<8)|((z&0xf)<<4)|(x&0xf) f = te_convert.get(id.lower(), lambda arg: (None, None, None)) # Do nothing if not found block, p2, meta = f(te) logger.debug('EntityInfoPre: ' +str(te)) logger.debug('EntityInfoPost: ' +' y='+str(y)+' z='+str(z)+' x='+str(x)+' Meta:'+str(meta)) # NB block and p2 never seems to be returned, but if this is important, then just change the above 'meta' to 'f(te)' if block != None: blocks[index] = block if p2 != None: param2[index] = p2 if meta != None: try: p = meta[0]["_plant"] if p > 15: content[index], param2[index] = conversion_table[941][p&0xf] else: content[index], param2[index] = conversion_table[940][p] except: self.metadata[(x&0xf, y&0xf, z&0xf)] = meta for e in mcblock.entities: id = e["id"] f = e_convert.get(id.lower(), lambda arg: (None, None, None)) # Do nothing if not found block, p2, meta = f(e) def save(self): os = BytesIO() writeU8(os, 25) # Version #flags flags = 0x00 if self.pos[1] < -1: flags |= 0x01 #is_underground flags |= 0x02 #day_night_differs flags |= 0x04 #lighting_expired flags |= 0x08 #generated writeU8(os, flags) writeU8(os, 2) # content_width writeU8(os, 2) # params_width cbuffer = BytesIO() # Bulk node data content = self.content k = 0 nimap = {} rev_nimap = [] first_free_content = 0 for z in range(16): for y in range(16): for x in range(16): #writeU16(cbuffer, content[k]) c = content[k] if c in nimap: writeU16(cbuffer, nimap[c]) else: nimap[c] = first_free_content writeU16(cbuffer, first_free_content) rev_nimap.append(c) first_free_content += 1 k += 1 k += (256-16) k += (16-16*256) param1 = self.param1 k = 0 for z in range(16): for y in range(16): for x in range(16): writeU8(cbuffer, param1[k]) k += 1 k += (256-16) k += (16-16*256) param2 = self.param2 k = 0 for z in range(16): for y in range(16): for x in range(16): writeU8(cbuffer, param2[k]) k += 1 k += (256-16) k += (16-16*256) os.write(zlib.compress(cbuffer.getvalue())) # Nodemeta meta = self.metadata cbuffer = BytesIO() writeU8(cbuffer, 1) # Version writeU16(cbuffer, len(meta)) for pos, data in meta.items(): writeU16(cbuffer, (pos[2]<<8)|(pos[1]<<4)|pos[0]) writeU32(cbuffer, len(data[0])) for name, val in data[0].items(): writeString(cbuffer, name) writeLongString(cbuffer, str(val)) serialize_inv(cbuffer, data[1]) os.write(zlib.compress(cbuffer.getvalue())) # Static objects writeU8(os, 0) # Version writeU16(os, 0) # Number of objects # Timestamp writeU32(os, 0xffffffff) # BLOCK_TIMESTAMP_UNDEFINED # Name-ID mapping writeU8(os, 0) # Version writeU16(os, len(rev_nimap)) for i in range(len(rev_nimap)): writeU16(os, i) writeString(os, self.name_id_mapping[rev_nimap[i]]) # Node timer writeU8(os, 2+4+4) # Timer data len writeU16(os, len(self.timers)) # Number of timers if len(self.timers) > 0: logger.info('wrote ' + str(len(self.timers)) + ' node timers') for i in range(len(self.timers)): writeU16(os, self.timers[i][0]) writeU32(os, self.timers[i][1]) writeU32(os, self.timers[i][2]) return os.getvalue() class MTMap: def __init__(self, path): self.world_path = path self.blocks = [] @staticmethod def getBlockAsInteger(p): return p[0]+4096*(p[1]+4096*p[2]) @staticmethod def fromMCMapBlocksIterator(mcmap, name_id_mapping, conversion_table): for mcblock in mcmap.getBlocksIterator(): mtblock = MTBlock(name_id_mapping) mtblock.fromMCBlock(mcblock, conversion_table) yield mtblock def fromMCMap(self, mcmap, nimap, ct): self.blocks = self.fromMCMapBlocksIterator(mcmap, nimap, ct) def save(self): conn = sqlite3.connect(os.path.join(self.world_path, "map.sqlite")) cur = conn.cursor() cur.execute("CREATE TABLE IF NOT EXISTS `blocks` (\ `pos` INT NOT NULL PRIMARY KEY, `data` BLOB);") num_saved = 0 for block in self.blocks: if num_saved%100 == 0: #print("Saved", num_saved, "blocks") conn.commit() num_saved += 1 cur.execute("INSERT INTO blocks VALUES (?,?)", # (self.getBlockAsInteger((-block.pos[0],block.pos[1],block.pos[2])), (self.getBlockAsInteger((-block.pos[0],block.pos[1],-block.pos[2])), block.save())) conn.commit() conn.close() if __name__ == "__main__": # Tests from random import randrange t = [randrange(256) for i in range(2048*8)] assert(MCBlock.extract_slice(MCBlock.expand_half_bytes(t), 0) == MCBlock.extract_slice_half_bytes(t, 0)) from time import time t0 = time() s1 = MCBlock.extract_slice(MCBlock.expand_half_bytes(t), 1) print(time()-t0) t0 = time() s2 = MCBlock.extract_slice_half_bytes(t, 1) print(time()-t0) assert(s1 == s2)