codekingpro/portable-devtools

1"""2This module contains function to analyse dynamic library3headers to extract system information4 5Currently only for MacOSX6 7Library file on macosx system starts with Mach-O or Fat field.8This can be distinguish by first 32 bites and it is called magic number.9Proper value of magic number is with suffix _MAGIC. Suffix _CIGAM means10reversed bytes order.11Both fields can occur in two types: 32 and 64 bytes.12 13FAT field inform that this library contains few version of library14(typically for different types version). It contains15information where Mach-O headers starts.16 17Each section started with Mach-O header contains one library18(So if file starts with this field it contains only one version).19 20After filed Mach-O there are section fields.21Each of them starts with two fields:22cmd - magic number for this command23cmdsize - total size occupied by this section information.24 25In this case only sections LC_VERSION_MIN_MACOSX (for macosx 10.13 and earlier)26and LC_BUILD_VERSION (for macosx 10.14 and newer) are interesting,27because them contains information about minimal system version.28 29Important remarks:30- For fat files this implementation looks for maximum number version.31  It not check if it is 32 or 64 and do not compare it with currently built package.32  So it is possible to false report higher version that needed.33- All structures signatures are taken form macosx header files.34- I think that binary format will be more stable than `otool` output.35  and if apple introduce some changes both implementation will need to be updated.36- The system compile will set the deployment target no lower than37  11.0 for arm64 builds. For "Universal 2" builds use the x86_64 deployment38  target when the arm64 target is 11.0.39"""40 41from __future__ import annotations42 43import ctypes44import os45import sys46 47"""here the needed const and struct from mach-o header files"""48 49FAT_MAGIC = 0xCAFEBABE50FAT_CIGAM = 0xBEBAFECA51FAT_MAGIC_64 = 0xCAFEBABF52FAT_CIGAM_64 = 0xBFBAFECA53MH_MAGIC = 0xFEEDFACE54MH_CIGAM = 0xCEFAEDFE55MH_MAGIC_64 = 0xFEEDFACF56MH_CIGAM_64 = 0xCFFAEDFE57 58LC_VERSION_MIN_MACOSX = 0x2459LC_BUILD_VERSION = 0x3260 61CPU_TYPE_ARM64 = 0x0100000C62 63mach_header_fields = [64    ("magic", ctypes.c_uint32),65    ("cputype", ctypes.c_int),66    ("cpusubtype", ctypes.c_int),67    ("filetype", ctypes.c_uint32),68    ("ncmds", ctypes.c_uint32),69    ("sizeofcmds", ctypes.c_uint32),70    ("flags", ctypes.c_uint32),71]72"""73struct mach_header {74    uint32_t	magic;		/* mach magic number identifier */75    cpu_type_t	cputype;	/* cpu specifier */76    cpu_subtype_t	cpusubtype;	/* machine specifier */77    uint32_t	filetype;	/* type of file */78    uint32_t	ncmds;		/* number of load commands */79    uint32_t	sizeofcmds;	/* the size of all the load commands */80    uint32_t	flags;		/* flags */81};82typedef integer_t cpu_type_t;83typedef integer_t cpu_subtype_t;84"""85 86mach_header_fields_64 = mach_header_fields + [("reserved", ctypes.c_uint32)]87"""88struct mach_header_64 {89    uint32_t	magic;		/* mach magic number identifier */90    cpu_type_t	cputype;	/* cpu specifier */91    cpu_subtype_t	cpusubtype;	/* machine specifier */92    uint32_t	filetype;	/* type of file */93    uint32_t	ncmds;		/* number of load commands */94    uint32_t	sizeofcmds;	/* the size of all the load commands */95    uint32_t	flags;		/* flags */96    uint32_t	reserved;	/* reserved */97};98"""99 100fat_header_fields = [("magic", ctypes.c_uint32), ("nfat_arch", ctypes.c_uint32)]101"""102struct fat_header {103    uint32_t	magic;		/* FAT_MAGIC or FAT_MAGIC_64 */104    uint32_t	nfat_arch;	/* number of structs that follow */105};106"""107 108fat_arch_fields = [109    ("cputype", ctypes.c_int),110    ("cpusubtype", ctypes.c_int),111    ("offset", ctypes.c_uint32),112    ("size", ctypes.c_uint32),113    ("align", ctypes.c_uint32),114]115"""116struct fat_arch {117    cpu_type_t	cputype;	/* cpu specifier (int) */118    cpu_subtype_t	cpusubtype;	/* machine specifier (int) */119    uint32_t	offset;		/* file offset to this object file */120    uint32_t	size;		/* size of this object file */121    uint32_t	align;		/* alignment as a power of 2 */122};123"""124 125fat_arch_64_fields = [126    ("cputype", ctypes.c_int),127    ("cpusubtype", ctypes.c_int),128    ("offset", ctypes.c_uint64),129    ("size", ctypes.c_uint64),130    ("align", ctypes.c_uint32),131    ("reserved", ctypes.c_uint32),132]133"""134struct fat_arch_64 {135    cpu_type_t	cputype;	/* cpu specifier (int) */136    cpu_subtype_t	cpusubtype;	/* machine specifier (int) */137    uint64_t	offset;		/* file offset to this object file */138    uint64_t	size;		/* size of this object file */139    uint32_t	align;		/* alignment as a power of 2 */140    uint32_t	reserved;	/* reserved */141};142"""143 144segment_base_fields = [("cmd", ctypes.c_uint32), ("cmdsize", ctypes.c_uint32)]145"""base for reading segment info"""146 147segment_command_fields = [148    ("cmd", ctypes.c_uint32),149    ("cmdsize", ctypes.c_uint32),150    ("segname", ctypes.c_char * 16),151    ("vmaddr", ctypes.c_uint32),152    ("vmsize", ctypes.c_uint32),153    ("fileoff", ctypes.c_uint32),154    ("filesize", ctypes.c_uint32),155    ("maxprot", ctypes.c_int),156    ("initprot", ctypes.c_int),157    ("nsects", ctypes.c_uint32),158    ("flags", ctypes.c_uint32),159]160"""161struct segment_command { /* for 32-bit architectures */162    uint32_t	cmd;		/* LC_SEGMENT */163    uint32_t	cmdsize;	/* includes sizeof section structs */164    char		segname[16];	/* segment name */165    uint32_t	vmaddr;		/* memory address of this segment */166    uint32_t	vmsize;		/* memory size of this segment */167    uint32_t	fileoff;	/* file offset of this segment */168    uint32_t	filesize;	/* amount to map from the file */169    vm_prot_t	maxprot;	/* maximum VM protection */170    vm_prot_t	initprot;	/* initial VM protection */171    uint32_t	nsects;		/* number of sections in segment */172    uint32_t	flags;		/* flags */173};174typedef int vm_prot_t;175"""176 177segment_command_fields_64 = [178    ("cmd", ctypes.c_uint32),179    ("cmdsize", ctypes.c_uint32),180    ("segname", ctypes.c_char * 16),181    ("vmaddr", ctypes.c_uint64),182    ("vmsize", ctypes.c_uint64),183    ("fileoff", ctypes.c_uint64),184    ("filesize", ctypes.c_uint64),185    ("maxprot", ctypes.c_int),186    ("initprot", ctypes.c_int),187    ("nsects", ctypes.c_uint32),188    ("flags", ctypes.c_uint32),189]190"""191struct segment_command_64 { /* for 64-bit architectures */192    uint32_t	cmd;		/* LC_SEGMENT_64 */193    uint32_t	cmdsize;	/* includes sizeof section_64 structs */194    char		segname[16];	/* segment name */195    uint64_t	vmaddr;		/* memory address of this segment */196    uint64_t	vmsize;		/* memory size of this segment */197    uint64_t	fileoff;	/* file offset of this segment */198    uint64_t	filesize;	/* amount to map from the file */199    vm_prot_t	maxprot;	/* maximum VM protection */200    vm_prot_t	initprot;	/* initial VM protection */201    uint32_t	nsects;		/* number of sections in segment */202    uint32_t	flags;		/* flags */203};204"""205 206version_min_command_fields = segment_base_fields + [207    ("version", ctypes.c_uint32),208    ("sdk", ctypes.c_uint32),209]210"""211struct version_min_command {212    uint32_t	cmd;		/* LC_VERSION_MIN_MACOSX or213                               LC_VERSION_MIN_IPHONEOS or214                               LC_VERSION_MIN_WATCHOS or215                               LC_VERSION_MIN_TVOS */216    uint32_t	cmdsize;	/* sizeof(struct min_version_command) */217    uint32_t	version;	/* X.Y.Z is encoded in nibbles xxxx.yy.zz */218    uint32_t	sdk;		/* X.Y.Z is encoded in nibbles xxxx.yy.zz */219};220"""221 222build_version_command_fields = segment_base_fields + [223    ("platform", ctypes.c_uint32),224    ("minos", ctypes.c_uint32),225    ("sdk", ctypes.c_uint32),226    ("ntools", ctypes.c_uint32),227]228"""229struct build_version_command {230    uint32_t	cmd;		/* LC_BUILD_VERSION */231    uint32_t	cmdsize;	/* sizeof(struct build_version_command) plus */232                                /* ntools * sizeof(struct build_tool_version) */233    uint32_t	platform;	/* platform */234    uint32_t	minos;		/* X.Y.Z is encoded in nibbles xxxx.yy.zz */235    uint32_t	sdk;		/* X.Y.Z is encoded in nibbles xxxx.yy.zz */236    uint32_t	ntools;		/* number of tool entries following this */237};238"""239 240 241def swap32(x):242    return (243        ((x << 24) & 0xFF000000)244        | ((x << 8) & 0x00FF0000)245        | ((x >> 8) & 0x0000FF00)246        | ((x >> 24) & 0x000000FF)247    )248 249 250def get_base_class_and_magic_number(lib_file, seek=None):251    if seek is None:252        seek = lib_file.tell()253    else:254        lib_file.seek(seek)255    magic_number = ctypes.c_uint32.from_buffer_copy(256        lib_file.read(ctypes.sizeof(ctypes.c_uint32))257    ).value258 259    # Handle wrong byte order260    if magic_number in [FAT_CIGAM, FAT_CIGAM_64, MH_CIGAM, MH_CIGAM_64]:261        if sys.byteorder == "little":262            BaseClass = ctypes.BigEndianStructure263        else:264            BaseClass = ctypes.LittleEndianStructure265 266        magic_number = swap32(magic_number)267    else:268        BaseClass = ctypes.Structure269 270    lib_file.seek(seek)271    return BaseClass, magic_number272 273 274def read_data(struct_class, lib_file):275    return struct_class.from_buffer_copy(lib_file.read(ctypes.sizeof(struct_class)))276 277 278def extract_macosx_min_system_version(path_to_lib):279    with open(path_to_lib, "rb") as lib_file:280        BaseClass, magic_number = get_base_class_and_magic_number(lib_file, 0)281        if magic_number not in [FAT_MAGIC, FAT_MAGIC_64, MH_MAGIC, MH_MAGIC_64]:282            return283 284        if magic_number in [FAT_MAGIC, FAT_CIGAM_64]:285 286            class FatHeader(BaseClass):287                _fields_ = fat_header_fields288 289            fat_header = read_data(FatHeader, lib_file)290            if magic_number == FAT_MAGIC:291 292                class FatArch(BaseClass):293                    _fields_ = fat_arch_fields294 295            else:296 297                class FatArch(BaseClass):298                    _fields_ = fat_arch_64_fields299 300            fat_arch_list = [301                read_data(FatArch, lib_file) for _ in range(fat_header.nfat_arch)302            ]303 304            versions_list = []305            for el in fat_arch_list:306                try:307                    version = read_mach_header(lib_file, el.offset)308                    if version is not None:309                        if el.cputype == CPU_TYPE_ARM64 and len(fat_arch_list) != 1:310                            # Xcode will not set the deployment target below 11.0.0311                            # for the arm64 architecture. Ignore the arm64 deployment312                            # in fat binaries when the target is 11.0.0, that way313                            # the other architectures can select a lower deployment314                            # target.315                            # This is safe because there is no arm64 variant for316                            # macOS 10.15 or earlier.317                            if version == (11, 0, 0):318                                continue319                        versions_list.append(version)320                except ValueError:321                    pass322 323            if len(versions_list) > 0:324                return max(versions_list)325            else:326                return None327 328        else:329            try:330                return read_mach_header(lib_file, 0)331            except ValueError:332                """when some error during read library files"""333                return None334 335 336def read_mach_header(lib_file, seek=None):337    """338    This function parses a Mach-O header and extracts339    information about the minimal macOS version.340 341    :param lib_file: reference to opened library file with pointer342    """343    base_class, magic_number = get_base_class_and_magic_number(lib_file, seek)344    arch = "32" if magic_number == MH_MAGIC else "64"345 346    class SegmentBase(base_class):347        _fields_ = segment_base_fields348 349    if arch == "32":350 351        class MachHeader(base_class):352            _fields_ = mach_header_fields353 354    else:355 356        class MachHeader(base_class):357            _fields_ = mach_header_fields_64358 359    mach_header = read_data(MachHeader, lib_file)360    for _i in range(mach_header.ncmds):361        pos = lib_file.tell()362        segment_base = read_data(SegmentBase, lib_file)363        lib_file.seek(pos)364        if segment_base.cmd == LC_VERSION_MIN_MACOSX:365 366            class VersionMinCommand(base_class):367                _fields_ = version_min_command_fields368 369            version_info = read_data(VersionMinCommand, lib_file)370            return parse_version(version_info.version)371        elif segment_base.cmd == LC_BUILD_VERSION:372 373            class VersionBuild(base_class):374                _fields_ = build_version_command_fields375 376            version_info = read_data(VersionBuild, lib_file)377            return parse_version(version_info.minos)378        else:379            lib_file.seek(pos + segment_base.cmdsize)380            continue381 382 383def parse_version(version):384    x = (version & 0xFFFF0000) >> 16385    y = (version & 0x0000FF00) >> 8386    z = version & 0x000000FF387    return x, y, z388 389 390def calculate_macosx_platform_tag(archive_root, platform_tag):391    """392    Calculate proper macosx platform tag basing on files which are included to wheel393 394    Example platform tag `macosx-10.14-x86_64`395    """396    prefix, base_version, suffix = platform_tag.split("-")397    base_version = tuple(int(x) for x in base_version.split("."))398    base_version = base_version[:2]399    if base_version[0] > 10:400        base_version = (base_version[0], 0)401    assert len(base_version) == 2402    if "MACOSX_DEPLOYMENT_TARGET" in os.environ:403        deploy_target = tuple(404            int(x) for x in os.environ["MACOSX_DEPLOYMENT_TARGET"].split(".")405        )406        deploy_target = deploy_target[:2]407        if deploy_target[0] > 10:408            deploy_target = (deploy_target[0], 0)409        if deploy_target < base_version:410            sys.stderr.write(411                "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value ({}) than "412                "the version on which the Python interpreter was compiled ({}), and "413                "will be ignored.\n".format(414                    ".".join(str(x) for x in deploy_target),415                    ".".join(str(x) for x in base_version),416                )417            )418        else:419            base_version = deploy_target420 421    assert len(base_version) == 2422    start_version = base_version423    versions_dict = {}424    for dirpath, _dirnames, filenames in os.walk(archive_root):425        for filename in filenames:426            if filename.endswith(".dylib") or filename.endswith(".so"):427                lib_path = os.path.join(dirpath, filename)428                min_ver = extract_macosx_min_system_version(lib_path)429                if min_ver is not None:430                    min_ver = min_ver[0:2]431                    if min_ver[0] > 10:432                        min_ver = (min_ver[0], 0)433                    versions_dict[lib_path] = min_ver434 435    if len(versions_dict) > 0:436        base_version = max(base_version, max(versions_dict.values()))437 438    # macosx platform tag do not support minor bugfix release439    fin_base_version = "_".join([str(x) for x in base_version])440    if start_version < base_version:441        problematic_files = [k for k, v in versions_dict.items() if v > start_version]442        problematic_files = "\n".join(problematic_files)443        if len(problematic_files) == 1:444            files_form = "this file"445        else:446            files_form = "these files"447        error_message = (448            "[WARNING] This wheel needs a higher macOS version than {}  "449            "To silence this warning, set MACOSX_DEPLOYMENT_TARGET to at least "450            + fin_base_version451            + " or recreate "452            + files_form453            + " with lower "454            "MACOSX_DEPLOYMENT_TARGET:  \n" + problematic_files455        )456 457        if "MACOSX_DEPLOYMENT_TARGET" in os.environ:458            error_message = error_message.format(459                "is set in MACOSX_DEPLOYMENT_TARGET variable."460            )461        else:462            error_message = error_message.format(463                "the version your Python interpreter is compiled against."464            )465 466        sys.stderr.write(error_message)467 468    platform_tag = prefix + "_" + fin_base_version + "_" + suffix469    return platform_tag470