From mboxrd@z Thu Jan  1 00:00:00 1970
Received: from eggs.gnu.org ([2001:4830:134:3::10]:48877)
	by lists.gnu.org with esmtp (Exim 4.71)
	(envelope-from <sanidhya.iiith@gmail.com>) id 1XSnsq-0000FV-GR
	for qemu-devel@nongnu.org; Sat, 13 Sep 2014 10:01:07 -0400
Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71)
	(envelope-from <sanidhya.iiith@gmail.com>) id 1XSnsc-0002EW-Mj
	for qemu-devel@nongnu.org; Sat, 13 Sep 2014 10:01:00 -0400
Received: from mail-yk0-x230.google.com ([2607:f8b0:4002:c07::230]:55477)
	by eggs.gnu.org with esmtp (Exim 4.71)
	(envelope-from <sanidhya.iiith@gmail.com>) id 1XSnsc-0002EJ-Fv
	for qemu-devel@nongnu.org; Sat, 13 Sep 2014 10:00:46 -0400
Received: by mail-yk0-f176.google.com with SMTP id 131so1097258ykp.7
	for <qemu-devel@nongnu.org>; Sat, 13 Sep 2014 07:00:46 -0700 (PDT)
From: Sanidhya Kashyap <sanidhya.iiith@gmail.com>
Date: Sat, 13 Sep 2014 10:00:14 -0400
Message-Id: <1410616814-27388-7-git-send-email-sanidhya.iiith@gmail.com>
In-Reply-To: <1410616814-27388-1-git-send-email-sanidhya.iiith@gmail.com>
References: <1410616814-27388-1-git-send-email-sanidhya.iiith@gmail.com>
Subject: [Qemu-devel] [PATCH v6 6/6] BitmapLog: python script for extracting
	bitmap from a binary file
List-Id: <qemu-devel.nongnu.org>
List-Unsubscribe: <https://lists.nongnu.org/mailman/options/qemu-devel>,
	<mailto:qemu-devel-request@nongnu.org?subject=unsubscribe>
List-Archive: <http://lists.nongnu.org/archive/html/qemu-devel>
List-Post: <mailto:qemu-devel@nongnu.org>
List-Help: <mailto:qemu-devel-request@nongnu.org?subject=help>
List-Subscribe: <https://lists.nongnu.org/mailman/listinfo/qemu-devel>,
	<mailto:qemu-devel-request@nongnu.org?subject=subscribe>
To: qemu list <qemu-devel@nongnu.org>
Cc: Sanidhya Kashyap <sanidhya.iiith@gmail.com>, "Dr. David Alan Gilbert" <dgilbert@redhat.com>, Juan Quintela <quintela@redhat.com>

I have modified the script to support the dump of the images to the file. Earlier,
everything was saved to the memory and later the processing was taking place. Now,
I have tried to solve that issue with only using the required memory.

After discussion with David, I have tried to select a base 2 matrix dimension like
512 X 512 or 1024 X 512 etc for the dumping of the bitmap. But, I am still supporting
the default square root based method of dimension selection.


Signed-off-by: Sanidhya Kashyap <sanidhya.iiith@gmail.com>
---

 scripts/extract-bitmap.py | 213 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 213 insertions(+)
 create mode 100755 scripts/extract-bitmap.py

diff --git a/scripts/extract-bitmap.py b/scripts/extract-bitmap.py
new file mode 100755
index 0000000..9a5a481
--- /dev/null
+++ b/scripts/extract-bitmap.py
@@ -0,0 +1,213 @@
+#!/usr/bin/python
+# This python script helps in extracting the dirty bitmap present
+# in the file after executing the log-dirty-bitmap command either
+# from the qmp or hmp interface. This file only processes binary
+# file obtained via command.
+#
+# Copyright (C) 2014 Sanidhya Kashyap <sanidhya.iiith@gmail.com>
+#
+# Authors:
+#       Sanidhya Kashyap
+#
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+
+import struct
+import argparse
+from functools import partial
+from math import sqrt
+from numpy import array
+from pylab import figure, imshow, savefig, gray, xlim, ylim
+from os import path, makedirs
+
+long_bytes = 8
+byte_size = 8
+int_bytes = 4
+block_list = []
+total_blocks = 0
+
+def get_unsigned_long_integer(value):
+	return struct.unpack('<Q', value)[0]
+
+def get_long_integer(value):
+	return struct.unpack('<q', value)[0]
+
+def get_integer(value):
+	return struct.unpack('<i', value)[0]
+
+def get_char(value):
+	return struct.unpack('<c', value)[0]
+
+def get_string(value, length):
+	name = struct.unpack('<'+str(length)+'s', value)[0]
+	for i in range(len(name)):
+		if name[i] == '\x00':
+			return name[:i]
+
+def dec2bin(decimal):
+    bin_value = bin(decimal)[2:]
+    if len(bin_value) < long_bytes * byte_size:
+        add_zeroes = long_bytes * byte_size - len(bin_value)
+        for i in range(add_zeroes):
+            bin_value += "0"
+    return str(bin_value)
+
+def get_bitmap_length(ram_bitmap_pages):
+    bitmap_length = ram_bitmap_pages / (long_bytes * byte_size)
+    if ram_bitmap_pages % (long_bytes * byte_size) != 0:
+        bitmap_length += 1
+    return bitmap_length
+
+def get_block_info():
+    print "Select any one of the following:"
+    for i in range(len(block_list)):
+        print str(i) + " " + block_list[i]['name']
+    return int(raw_input('Enter the number: '))
+
+def get_matrix(all_digits, y, x):
+    v = []
+    xlim(xmin = 0, xmax = x)
+    ylim(ymin = 0, ymax = y)
+    for i in range(y):
+        v1 = []
+        for j in range(x):
+            v1.append(int(all_digits[i * x + j]))
+        v.append(v1)
+    return v
+
+def get_matrix_base2(all_digits):
+    l = len(all_digits)
+    sqrtvalue = int(sqrt(l))
+    x = 2 ** (sqrtvalue.bit_length() - 1)
+    y = x * 2
+    if (x * y - l < 0):
+        x = y
+    for i in range(x * y - l):
+        all_digits += "0"
+
+    return get_matrix(all_digits, x, y);
+
+def get_matrix_sqrt(all_digits):
+    l = len(all_digits)
+    sqrtvalue = int(sqrt(l))
+    for i in range(sqrtvalue * (sqrtvalue + 1) - l):
+        all_digits += "0"
+
+    if sqrtvalue * sqrtvalue == l:
+        return get_matrix(all_digits, sqrtvalue, sqrtvalue)
+    else:
+        return get_matrix(all_digits, sqrtvalue, sqrtvalue + 1)
+
+def dump_ram_block_info(infile):
+    total_blocks = get_integer(infile.read(int_bytes))
+    for i in range(total_blocks):
+        block_name_length = get_integer(infile.read(int_bytes))
+        block_name = get_string(infile.read(block_name_length), block_name_length)
+        block_offset = get_unsigned_long_integer(infile.read(long_bytes))
+        block_length = get_unsigned_long_integer(infile.read(long_bytes))
+        block_list.append(dict(name=block_name, offset=block_offset, length=block_length))
+
+def generate_image(all_digits, num, debug, dir, base):
+    v = []
+
+    if base is False:
+        v = get_matrix_sqrt(all_digits)
+    else:
+        v = get_matrix_base2(all_digits)
+
+    im_array = array(v)
+    figure(num)
+    imshow(im_array, cmap=gray())
+
+    filename=dir + "/" + "out_" + str(num) + ".png"
+    del v
+    savefig(filename)
+    if debug is True:
+        print 'file ' + filename + ' dumped'
+
+
+def dump_bitmap(args):
+    marker = 'M'
+    count = 0
+    value = ' '
+    block_offset = 0
+    block_length = 0
+    block_num = 0
+    current_ram_bitmap_pages = 0
+    prev_ram_bitmap_pages = 0
+    infile = open(format(args.infile), 'rb')
+    debug = args.debug
+    dump_all = args.dump_all
+    dir = args.dir
+    base = args.base
+
+    if not path.exists(dir):
+        makedirs(dir)
+
+    while True:
+        if len(value) == 0  or marker != 'M':
+            print "issue with the dump"
+            return
+        bitmap_page_raw_value = infile.read(long_bytes)
+        if not bitmap_page_raw_value:
+            break
+        current_ram_bitmap_pages = get_long_integer(bitmap_page_raw_value)
+        if current_ram_bitmap_pages != prev_ram_bitmap_pages:
+            prev_ram_bitmap_pages = current_ram_bitmap_pages
+            dump_ram_block_info(infile)
+            # asking what should be printed
+            if dump_all is False:
+                if count == 0:
+                    block_num = get_block_info()
+                    if debug is True:
+                        print block_list[block_num]['name'] + ' selected'
+                else:
+                    x = block_num
+                    y = len(block_list) / total_blocks
+                    block_num = total_blocks * (y-1) + x
+                block_offset = block_list[block_num]['offset']
+                block_length = block_list[block_num]['length']
+                if debug is True:
+                    print 'total ram bitmap pages: ' + str(current_ram_bitmap_pages)
+                    print block_list[block_num]['name'] + ' offset: ' + str(block_offset)
+                    print block_list[block_num]['name'] + ' length: ' + str(block_length)
+
+        bitmap_length = get_bitmap_length(current_ram_bitmap_pages)
+        bitmap_raw_value = infile.read(long_bytes * bitmap_length)
+        if not bitmap_raw_value:
+            break
+
+        count+=1
+        all_digits=""
+        for i in range(bitmap_length):
+            mark = i * long_bytes
+            all_digits += dec2bin(get_unsigned_long_integer(bitmap_raw_value[mark : mark + long_bytes]))
+
+        if dump_all is False:
+            generate_image(all_digits[block_offset : block_offset + block_length], count, debug, dir, base)
+        else:
+            generate_image(all_digits, count, debug, dir, base)
+        value = infile.read(1)
+        marker = get_char(value)
+    infile.close()
+
+def main():
+    extracter = argparse.ArgumentParser(description='Extract dirty bitmap from binary file.')
+    extracter.add_argument('-f', '--file', dest='infile',
+            help='Input file to extract the bitmap', metavar='FILE')
+    extracter.add_argument('-a', '--all', action='store_true', dest='dump_all', default=False,
+            help='Use all memory blocks for the figure')
+    extracter.add_argument('-d', '--debug', action='store_true', dest='debug', default=False,
+            help='print the debug info')
+    extracter.add_argument('-dir', dest='dir', default='.',
+            help='directory to store the image files')
+    extracter.add_argument('-b', '--base2', dest='base', action='store_true', default=False,
+            help='dump the bitmap array in powers of 2')
+
+    args = extracter.parse_args()
+    print 'The filename is {}'.format(args.infile)
+
+    dump_bitmap(args)
+
+if __name__ == '__main__':
+    main()
-- 
1.9.1