From a3ab58b99e423c62ea511319cbe2fddd0200cb05 Mon Sep 17 00:00:00 2001 From: yijian Date: Thu, 26 Aug 2021 15:36:25 +0800 Subject: [PATCH] Added to support sparse image format Signed-off-by: yijian --- build_scripts/parse_params.sh | 10 -- ohos/images/mkimage/covert.py | 221 +++++++++++++++++++++++++++++ ohos/images/mkimage/mkextimage.py | 4 +- ohos/images/mkimage/mkf2fsimage.py | 4 - ohos/images/mkimage/mkimages.py | 6 +- 5 files changed, 225 insertions(+), 20 deletions(-) create mode 100755 ohos/images/mkimage/covert.py diff --git a/build_scripts/parse_params.sh b/build_scripts/parse_params.sh index 46674cf875..ca1f40d094 100755 --- a/build_scripts/parse_params.sh +++ b/build_scripts/parse_params.sh @@ -95,13 +95,3 @@ if [[ "${use_ccache}" == true ]]; then set_ccache set -e fi - -if [[ "${sparse_image}" == true ]]; then - set +e - which img2simg > /dev/null 2>&1 - if [[ $? -ne 0 ]]; then - echo -e "\033[31mError: the img2simg is not available, please install img2simg.\033[0m" - exit 1 - fi - set -e -fi diff --git a/ohos/images/mkimage/covert.py b/ohos/images/mkimage/covert.py new file mode 100755 index 0000000000..3d65d3a7dd --- /dev/null +++ b/ohos/images/mkimage/covert.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2021 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Author: @easy-to-see + +#ImageFormat： +# MagicNumber +# Version +# BlockSize +# TotalBlocks +# CRCValueOfRawFile +# CRCValueOfBlockTab +# TableNumers +# BStart1 Start of 0 +# BEnd1 not include end block +# BStart2 +# BEnd2 +# ……………… +# gapforblockSize +# gap of 1 BlockSize +# block data for BStart1 +# block data for BStart1 + 1 +# ……………… +# block data for BEnd1 - 1 +# block data for BStart2 +# ……………… + +import sys +import os +import hashlib +import errno + + +def usage(): + print('\n Usage: covert.py ') + print(' : sparse or unsparse') + print(' : input image file') + print(' : ouput image file\n') + return + + +def getGAPBlockSize(length, size): +# MUST have 1+ block gap + if length < size: + cnt = 2 + elif length < (size * 2): + cnt = 3 + else: + cnt = 4 + # print("CNT:",length, size, cnt) + return cnt + + +def getBlockCnt(inputFile, BlockSize): + size = os.path.getsize(inputFile) + TotalBlocks = size / BlockSize + if (size % BlockSize) != 0: + print("len is not eq n * Blocksize: ", size, TotalBlocks) + sys.exit(1) + return TotalBlocks + + +def getCRCvalue(inputFile, BlockSize): + TotalBlocks = getBlockCnt(inputFile, BlockSize) + indata = open(inputFile, 'rb') + ind = 0 + md5 = hashlib.md5() + while (ind < TotalBlocks): + md5.update(indata.read(BlockSize)) + ind += 1 + indata.close() + return md5.hexdigest() + + +def unsparse(SparseImageFile, ImageFile): + header = open(SparseImageFile, 'r') + MagicNumber = int(header.readline()) + Version = int(header.readline()) + BlockSize = int(header.readline()) + TotalBlocks = int(header.readline()) + CRCValue = header.readline() + inputCRCValue = header.readline() + TableNumbers = int(header.readline()) + Table = [] + i = 0 + while (i < TableNumbers): + start = int(header.readline()) + end = int(header.readline()) + Table.append([start, end]) + i += 1 + length = header.tell() + header.close() + + inputRow = open(SparseImageFile, 'rb') + inputRow.seek(getGAPBlockSize(length, BlockSize) * BlockSize) + output = open(ImageFile, 'wb') + output.truncate(TotalBlocks * BlockSize) + + md5 = hashlib.md5() + for block in Table: + cnt = block[1] - block[0] + output.seek(block[0] * BlockSize) + indata = inputRow.read(cnt * BlockSize) + md5.update(indata) + output.write(indata) + + output.close() + inputRow.close() + + print("RawFileCRC: ", getCRCvalue(ImageFile, BlockSize), CRCValue) + print("SparseCRC: ", md5.hexdigest(), inputCRCValue) + return + + +def isEmptyBlock(buff, size): + ind = 0 + while (ind < size): + if buff[ind] != 0: + return False + ind += 1 + return True + + +def getRawDataFile(ImageFile, BlockID, TotalBlocks, BlockSize): + TEMP_FILE = ImageFile + ".tempfile" + ind = 0 + start = -1 + TableNumbers = 0 + + inputRow = open(ImageFile, 'rb') + outputtemp = open(TEMP_FILE, 'wb') + while (ind < TotalBlocks): + indata = inputRow.read(BlockSize) + if len(indata) != BlockSize: + print("error Block", ind, len(indata)) + if isEmptyBlock(indata, BlockSize) == True: + if start != -1: + BlockID.append([start, ind]) + TableNumbers += 1 + start = -1 + else: + # print("ind:", ind) + outputtemp.write(indata) + if start == -1: + start = ind + ind += 1 + if start != -1: + BlockID.append([start, ind]) + TableNumbers += 1 + start = -1 + inputRow.close() + outputtemp.close() + return TableNumbers + + +def sparse(ImageFile, SparseImageFile): + TEMP_FILE = ImageFile + ".tempfile" + MagicNumber = 12344321 + Version = 1 + BlockSize = 4096 + TableNumbers = 0 + BlockID = [] + + TotalBlocks = getBlockCnt(ImageFile, BlockSize) + TableNumbers = getRawDataFile(ImageFile, BlockID, TotalBlocks, BlockSize) + +# save the header + outputRow = open(SparseImageFile, 'w') + outputRow.write("%s\n" % (MagicNumber)) + outputRow.write("%s\n" % (Version)) + outputRow.write("%s\n" % (BlockSize)) + outputRow.write("%s\n" % (int(TotalBlocks))) + outputRow.write("%s\n" % (getCRCvalue(ImageFile, BlockSize))) + outputRow.write("%s\n" % (getCRCvalue(TEMP_FILE, BlockSize))) + outputRow.write("%s\n" % (TableNumbers)) + for block in BlockID: + outputRow.write("%s\n" % (block[0])) + outputRow.write("%s\n" % (block[1])) + outputRow.truncate(getGAPBlockSize(outputRow.tell(), BlockSize) * BlockSize) + outputRow.close() + +# append the raw data + outputRow = open(SparseImageFile, 'ab') + outputtemp = open(TEMP_FILE, 'rb') + blocknum = getBlockCnt(TEMP_FILE, BlockSize) + i = 0 + while (i < blocknum): + outputRow.write(outputtemp.read(BlockSize)) + i += 1 + outputtemp.close() + outputRow.close() + os.remove(TEMP_FILE) + + +if __name__ == '__main__': + try: + cmd = str(sys.argv[1]) + InputFile = str(sys.argv[2]) + OutputFile = str(sys.argv[3]) + except IndexError: + usage() + sys.exit() + + if cmd == 'unsparse': + unsparse(InputFile, OutputFile) + elif cmd == 'sparse': + sparse(InputFile, OutputFile) + else: + usage() diff --git a/ohos/images/mkimage/mkextimage.py b/ohos/images/mkimage/mkextimage.py index 2d1ff199de..b20b5ccae4 100755 --- a/ohos/images/mkimage/mkextimage.py +++ b/ohos/images/mkimage/mkextimage.py @@ -89,11 +89,9 @@ def build_run_mke2fs(args): def build_run_e2fsdroid(args): - e2fsdroid_opts = "" + e2fsdroid_opts = " -e" e2fsdroid_cmd = "" - if not args.extend_opts or not "sparse" in args.extend_opts: - e2fsdroid_opts += " -e" if args.dac_config: e2fsdroid_opts += " -C " + args.dac_config if args.file_context: diff --git a/ohos/images/mkimage/mkf2fsimage.py b/ohos/images/mkimage/mkf2fsimage.py index 4d53453f8b..5bafd74a9c 100755 --- a/ohos/images/mkimage/mkf2fsimage.py +++ b/ohos/images/mkimage/mkf2fsimage.py @@ -57,8 +57,6 @@ def build_run_mkf2fs(args): mkf2fs_opts = "" mkf2fs_cmd = "" - if args.sparse: - mkf2fs_opts += " -S " + args.fs_size if args.label: mkf2fs_opts += " -l " + args.label else: @@ -84,8 +82,6 @@ def build_run_sloadf2fs(args): sloadf2fs_opts = "" sloadf2fs_cmd = "" - if args.sparse: - sloadf2fs_opts += " -S" if args.dac_config: sloadf2fs_opts += " -C " + args.dac_config sloadf2fs_opts += " -f " + args.src_dir diff --git a/ohos/images/mkimage/mkimages.py b/ohos/images/mkimage/mkimages.py index 24b09a092b..2696a3960d 100755 --- a/ohos/images/mkimage/mkimages.py +++ b/ohos/images/mkimage/mkimages.py @@ -86,10 +86,10 @@ def mk_images(args): res[2].decode() + res[3].decode()) print("MkImages failed errno: " + str(res[1])) sys.exit(2) - # we dont support sparse in mktools. - if "sparse" in is_sparse: + + if "sparse" == is_sparse: tmp_device = device + ".tmp" - res = run_cmd("img2simg " + device + " " + tmp_device) + res = run_cmd("covert.py sparse " + device + " " + tmp_device) os.rename(tmp_device, device) -- Gitee