From 37f3f33bf2ab665341eaf3c6d2eba22011c8a0d9 Mon Sep 17 00:00:00 2001 From: Stefan Sauer Date: Thu, 18 Aug 2016 10:37:55 +0200 Subject: [PATCH 1/2] block.py: add progress reporting based on num_chunks We don't know the number of blocks on each chunk beforehand, but we know the number of chunks which is precise enough for a status report. Also use a wall clock to give an eta for the conversion. See #6 --- block.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/block.py b/block.py index 7cda649..ea0faa7 100644 --- a/block.py +++ b/block.py @@ -1,7 +1,9 @@ import os +import sys import zlib import nbt import random +import time from io import BytesIO import sqlite3 from serialize import * @@ -36,16 +38,29 @@ class MCMap: if bytesToInt(f.read(3)) != 0: self.chunk_pos.append((chkx, chkz)) chunkCountb += 1 -# print('Total Chunks: ' + str(len(self.chunk_pos))) def getChunk(self, chkx, chkz): return MCChunk(chkx, chkz, self.world_path, self.ext) def getBlocksIterator(self): + num_chunks = len(self.chunk_pos) + chunk_ix = 0 + t0 = time.time() for chkx, chkz in self.chunk_pos: + if chunk_ix%10 == 0: + if chunk_ix > 0: + td = time.time() - t0 # wall clock time spent + tr = ((num_chunks * td) / chunk_ix) - td # time remaining + eta = time.strftime("%H:%M:%S", time.gmtime(tr)) + else: + eta = "??:??:??" + print('Processed %d / %d chunks, ETA %s h:m:s' % (chunk_ix, num_chunks, eta), end='\r') + sys.stdout.flush() + chunk_ix += 1 blocks = self.getChunk(chkx, chkz).blocks for block in blocks: yield block + print() class MCChunk: """A 16x16 column of nodes""" @@ -414,8 +429,8 @@ class MTBlock: # Node timer writeU8(os, 2+4+4) # Timer data len writeU16(os, len(self.timers)) # Number of timers - if len(self.timers) > 0: - print('wrote ' + str(len(self.timers)) + ' node timers') + #if len(self.timers) > 0: + # print('wrote ' + str(len(self.timers)) + ' node timers') for i in range(len(self.timers)): writeU16(os, self.timers[i][0]) writeU32(os, self.timers[i][1]) @@ -452,7 +467,7 @@ class MTMap: num_saved = 0 for block in self.blocks: if num_saved%100 == 0: - print("Saved", num_saved, "blocks") + #print("Saved", num_saved, "blocks") conn.commit() num_saved += 1 cur.execute("INSERT INTO blocks VALUES (?,?)", From acd7eaa4446ede45136b10923325e08a00cd03a7 Mon Sep 17 00:00:00 2001 From: Stefan Sauer Date: Thu, 18 Aug 2016 10:40:15 +0200 Subject: [PATCH 2/2] block.py: use logging to report issues This way diagnostics are printed to stderr and can easilly be redirected to a file. Additionally we can filter information and don#t need to comment print statements. --- block.py | 24 +++++++++++++----------- entities.py | 16 ++++++++++------ mcimport.py | 3 +++ tile_entities.py | 5 ++++- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/block.py b/block.py index ea0faa7..0c86bde 100644 --- a/block.py +++ b/block.py @@ -4,6 +4,7 @@ import zlib import nbt import random import time +import logging from io import BytesIO import sqlite3 from serialize import * @@ -11,6 +12,7 @@ from itemstack import * from tile_entities import te_convert from entities import e_convert +logger = logging.getLogger('block') class MCMap: """A MC map""" @@ -229,7 +231,7 @@ class MTBlock: self.pos = (0, 0, 0) def fromMCBlock(self, mcblock, conversion_table): -# print('\n***fromMCBlock: Starting New Block***') + logger.debug('***fromMCBlock: Starting New Block***') self.timers = [] self.pos = (mcblock.pos[0], mcblock.pos[1]-4, mcblock.pos[2]) @@ -273,9 +275,9 @@ class MTBlock: elif isdoor(blocks[i]) and data[i] < 8: above = i + 256 if (above >= 4096): - print('Unable to fix door - top part is across block boundary!') + logger.warning('Unable to fix door - top part is across block boundary! (%d >= 4096)' % above) elif isdoor(blocks[above]) and data[above] < 7: - print('Unable to fix door - bottom part on top of bottom part!') + logger.warning('Unable to fix door - bottom part on top of bottom part!') else: d_right = data[above] & 1 # 0 - left, 1 - right d_open = data[i] & 4 # 0 - closed, 1 - open @@ -289,9 +291,9 @@ class MTBlock: elif isdoor(blocks[i]) and data[i] >= 8: below = i - 256 if (below < 0): - print('Unable to fix door - bottom part is across block boundary!') + logger.warning('Unable to fix door - bottom part is across block boundary! (%d < 0)' % below) elif isdoor(blocks[below]) and data[below] >= 8: - print('Unable to fix door - top part below top part!') + logger.warning('Unable to fix door - top part below top part!') else: d_right = data[i] & 1 # 0 - left, 1 - right d_open = data[below] & 4 # 0 - closed, 1 - open @@ -304,7 +306,7 @@ class MTBlock: self.metadata[(i & 0xf, (i>>8) & 0xf, (i>>4) & 0xf)] = ({ "right": "1" }, {}) elif content[i]==0 and param2[i]==0 and not (blocks[i]==0): - print('Unknown Minecraft Block:' + str(mcblockidentifier[i])) # This is the minecraft ID#/data as listed in map_content.txt + logger.warning('Unknown Minecraft Block:' + str(mcblockidentifier[i])) # This is the minecraft ID#/data as listed in map_content.txt for te in mcblock.tile_entities: id = te["id"] @@ -312,8 +314,8 @@ class MTBlock: index = ((y&0xf)<<8)|((z&0xf)<<4)|(x&0xf) f = te_convert.get(id.lower(), lambda arg: (None, None, None)) # Do nothing if not found block, p2, meta = f(te) -# print('\nEntityInfoPre: ' +str(te)) # EntityInfo: if you want to print pre-conversion entity information then uncomment this line -# print('EntityInfoPost: ' +' y='+str(y)+' z='+str(z)+' x='+str(x)+' Meta:'+str(meta)) # EntityInfo: if you want to print post-conversion entity information then uncomment this line + logger.debug('EntityInfoPre: ' +str(te)) + logger.debug('EntityInfoPost: ' +' y='+str(y)+' z='+str(z)+' x='+str(x)+' Meta:'+str(meta)) # NB block and p2 never seems to be returned, but if this is important, then just change the above 'meta' to 'f(te)' if block != None: @@ -330,7 +332,7 @@ class MTBlock: else: content[above], param2[above] = conversion_table[940][p] else: - print("can't pot plant in pot across block border, or not air") + logger.warning("can't pot plant in pot across block border, or not air") except: self.metadata[(x&0xf, y&0xf, z&0xf)] = meta @@ -429,8 +431,8 @@ class MTBlock: # Node timer writeU8(os, 2+4+4) # Timer data len writeU16(os, len(self.timers)) # Number of timers - #if len(self.timers) > 0: - # print('wrote ' + str(len(self.timers)) + ' node timers') + if len(self.timers) > 0: + logger.info('wrote ' + str(len(self.timers)) + ' node timers') for i in range(len(self.timers)): writeU16(os, self.timers[i][0]) writeU32(os, self.timers[i][1]) diff --git a/entities.py b/entities.py index a111e47..9c55389 100644 --- a/entities.py +++ b/entities.py @@ -1,10 +1,15 @@ +import logging from itemstack import * #If you wish to add more entities, then... -# To print out pre and post-conversion entity information uncomment line 237 (ish) in blocks.py (search for 'EntityInfo' to locate it) +# To see pre and post-conversion entity, raise the log level to DEBUG and look +# at the log output on stderr in category 'blocks' (search for 'EntityInfo' to +# locate them) + +logger = logging.getLogger('entities') def convert_frame(e): - from pprint import pprint - pprint(e) + from pprint import pformat + logger.debug(pformat(e)) # must read attribs and translate to entities we know from map_content.txt content = e.get("Item") x = e.get("TileX") @@ -12,11 +17,10 @@ def convert_frame(e): z = e.get("TileZ") item = e.get("Item") if item: - content = item.get("id") - print(content) + logger.debug(item.get("id")) return "xdecor:itemframe", None, (None, None) else: - print("empty item frame") + logger.warning("empty item frame") return "xdecor:itemframe", None, (None, None) e_convert = {"itemframe": convert_frame} diff --git a/mcimport.py b/mcimport.py index e1c3a3d..1333f25 100644 --- a/mcimport.py +++ b/mcimport.py @@ -2,9 +2,12 @@ import os import sys +import logging from block import * import content +logging.basicConfig(level=logging.INFO) + if (sys.version_info < (3, 0)): print("This script does not work with Python < 3.0, sorry.") exit(1) diff --git a/tile_entities.py b/tile_entities.py index 1721130..e675aca 100644 --- a/tile_entities.py +++ b/tile_entities.py @@ -1,7 +1,10 @@ +import logging from itemstack import * #If you wish to add more entities, then... # To print out pre and post-conversion entity information uncomment line 237 (ish) in blocks.py (search for 'EntityInfo' to locate it) +logger = logging.getLogger('tile_entities') + def convert_chest(te): formspec = "size[8,9]"+\ "list[current_name;main;0,0;8,4;]"+\ @@ -101,7 +104,7 @@ def convert_pot(te): fields = { "_plant": t[c] } return None, None, (fields, {}) except: - print('Unknown flower pot type: '+c) + logger.warning('Unknown flower pot type: ' + c) return None, None, None def convert_cmdblock(te):