# Copyright 2001-2026 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

import tarfile
import traceback
import io
import threading
import subprocess
import errno
import pwd
import grp
import re
import shlex
import stat
import sys
import tempfile
from copy import copy
from datetime import datetime

import portage
from portage import checksum
from portage import os
from portage import shutil
from portage import normalize_path
from portage import _encodings
from portage import _unicode_decode
from portage import _unicode_encode
from portage.binpkg import get_binpkg_format
from portage.exception import (
    FileNotFound,
    InvalidBinaryPackageFormat,
    InvalidCompressionMethod,
    CompressorNotFound,
    CompressorOperationFailed,
    CommandNotFound,
    GPGException,
    DigestException,
    MissingSignature,
    InvalidSignature,
    SignedPackage,
)
from portage.output import colorize, EOutput
from portage.util._urlopen import urlopen
from portage.util import writemsg
from portage.util import varexpand
from portage.util.compression_probe import _compressors
from portage.util.cpuinfo import makeopts_to_job_count
from portage.process import find_binary
from portage.const import MANIFEST2_HASH_DEFAULTS, HASHING_BLOCKSIZE


class tar_stream_writer:
    """
    One-pass helper function that return a file-like object
    for create a file inside of a tar container.

    This helper allows streaming add a new file to tar
    without prior knows the file size.

    With optional call and pipe data through external program,
    the helper can transparently save compressed data.

    With optional checksum helper, this helper can create
    corresponding checksum and GnuPG signature.

    Example:

    writer = tar_stream_writer(
            file_tarinfo,            # the file tarinfo that need to be added
            container,               # the outer container tarfile object
            tarfile.USTAR_FORMAT,    # the outer container format
            ["gzip"],                # compression command
            checksum_helper          # checksum helper
    )

    writer.write(data)
    writer.close()
    """

    def __init__(
        self,
        tarinfo,
        container,
        tar_format,
        cmd=None,
        checksum_helper=None,
        uid=None,
        gid=None,
    ):
        """
        tarinfo          # the file tarinfo that need to be added
        container        # the outer container tarfile object
        tar_format       # the outer container format for create the tar header
        cmd              # subprocess.Popen format compression command
        checksum_helper  # checksum helper
        uid              # drop root user to uid
        gid              # drop root group to gid
        """
        self.checksum_helper = checksum_helper
        self.cmd = cmd
        self.closed = False
        self.container = container
        self.killed = False
        self.error = False
        self.tar_format = tar_format
        self.tarinfo = tarinfo
        self.uid = uid
        self.gid = gid

        # Record container end position
        self.container.fileobj.seek(0, io.SEEK_END)
        self.begin_position = self.container.fileobj.tell()
        self.end_position = 0
        self.file_size = 0

        # Write tar header without size
        tar_header = self.tarinfo.tobuf(
            self.tar_format, self.container.encoding, self.container.errors
        )
        self.header_size = len(tar_header)
        self.container.fileobj.write(tar_header)
        self.container.fileobj.flush()
        self.container.offset += self.header_size

        # Start external compressor if needed
        if cmd is None:
            self.proc = None
        else:
            self.proc = subprocess.Popen(
                cmd,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                user=self.uid,
                group=self.gid,
            )

            self.read_thread = threading.Thread(
                target=self._cmd_read_thread, name="tar_stream_cmd_read", daemon=True
            )
            self.read_thread.start()

    def __del__(self):
        self.close()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        self.close()

    def kill(self):
        """
        Kill external program if any error happened in Python
        """
        if self.proc is not None:
            self.killed = True
            self.proc.kill()
            try:
                self.proc.stdin.close()
            except BrokenPipeError:
                traceback.print_exc()
            self.close()

    def _cmd_read_thread(self):
        """
        Use a thread to avoid block.
        Read stdout from external compressor, then write to the file
        in container, and to checksum helper if needed.
        """
        while True:
            try:
                buffer = self.proc.stdout.read(HASHING_BLOCKSIZE)
                if not buffer:
                    self.proc.stdout.close()
                    return
            except BrokenPipeError:
                self.proc.stdout.close()
                writemsg(colorize("BAD", f"GPKG subprocess failed: {self.cmd} \n"))
                if not self.killed:
                    # Do not raise error if killed by Portage
                    raise CompressorOperationFailed("PIPE broken")
            try:
                self.container.fileobj.write(buffer)
            except OSError as err:
                self.error = True
                self.kill()
                raise CompressorOperationFailed(str(err))
            if self.checksum_helper:
                self.checksum_helper.update(buffer)

    def write(self, data):
        """
        Write data to tarfile or external compressor stdin
        """
        if self.closed:
            raise OSError("writer closed")

        if self.proc:
            # Write to external program
            try:
                self.proc.stdin.write(data)
            except BrokenPipeError:
                self.error = True
                writemsg(colorize("BAD", f"GPKG subprocess failed: {self.cmd} \n"))
                raise CompressorOperationFailed("PIPE broken")
        else:
            # Write to container
            self.container.fileobj.write(data)
            if self.checksum_helper:
                self.checksum_helper.update(data)

    def close(self):
        """
        Update the new file's tar header on close
        """
        if self.closed:
            return

        # Wait for compressor exit
        if self.proc is not None:
            self.proc.stdin.close()
            if self.proc.wait() != os.EX_OK:
                if not (self.killed or self.error):
                    raise CompressorOperationFailed("compression failed")
            if self.read_thread.is_alive():
                self.read_thread.join()

        # Get container end position and calculate file size
        self.container.fileobj.seek(0, io.SEEK_END)
        self.end_position = self.container.fileobj.tell()
        self.file_size = self.end_position - self.begin_position - self.header_size
        self.tarinfo.size = self.file_size

        # Tar block is 512, need padding \0
        _, remainder = divmod(self.file_size, 512)
        if remainder > 0:
            padding_size = 512 - remainder
            self.container.fileobj.write(b"\0" * padding_size)
            self.container.offset += padding_size
            self.container.fileobj.flush()

        # Update tar header
        tar_header = self.tarinfo.tobuf(
            self.tar_format, self.container.encoding, self.container.errors
        )
        self.container.fileobj.seek(self.begin_position)
        self.container.fileobj.write(tar_header)
        self.container.fileobj.seek(0, io.SEEK_END)
        self.container.fileobj.flush()
        self.container.offset = self.container.fileobj.tell()
        self.closed = True

        # Add tarinfo to tarfile
        self.container.members.append(self.tarinfo)

        if self.checksum_helper:
            self.checksum_helper.finish()

        self.closed = True


class tar_stream_reader:
    """
    Helper function that returns a file-like object
    for reading a file inside of a tar container.

    This helper allows transparently streaming read a compressed
    file in tar.

    With optional call and pipe compressed data through external
    program, and return the uncompressed data.

    reader = tar_stream_reader(
            fileobj,             # the fileobj from tarfile.extractfile(f)
            ["gzip", "-d"],      # decompression command
    )

    reader.read()
    reader.close()
    """

    def __init__(self, fileobj, cmd=None, uid=None, gid=None):
        """
        fileobj should be a file-like object that has read().
        cmd is an optional external decompressor command.
        """
        self.closed = False
        self.cmd = cmd
        self.fileobj = fileobj
        self.killed = False
        self.uid = uid
        self.gid = gid

        if cmd is None:
            self.read_io = fileobj
            self.proc = None
        else:
            # Start external decompressor
            self.proc = subprocess.Popen(
                cmd,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                user=self.uid,
                group=self.gid,
            )
            self.read_io = self.proc.stdout
            # Start stdin block writing thread
            self.thread = threading.Thread(
                target=self._write_thread, name="tar_stream_stdin_writer", daemon=True
            )
            self.thread.start()

    def __del__(self):
        try:
            self.close()
        except CompressorOperationFailed:
            pass

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        try:
            self.close()
        except CompressorOperationFailed:
            pass

    def _write_thread(self):
        """
        Write thread to avoid full buffer blocking
        """
        try:
            while True:
                buffer = self.fileobj.read(HASHING_BLOCKSIZE)
                if buffer:
                    try:
                        self.proc.stdin.write(buffer)
                    except ValueError:
                        if self.killed:
                            return
                        else:
                            raise
                else:
                    self.proc.stdin.flush()
                    self.proc.stdin.close()
                    break
        except BrokenPipeError:
            if self.killed is False:
                writemsg(colorize("BAD", f"GPKG subprocess failed: {self.cmd} \n"))
                raise CompressorOperationFailed("PIPE broken")

    def kill(self):
        """
        Kill external program if any error happened in Python
        """
        if self.proc is not None:
            self.killed = True
            self.proc.kill()
            try:
                self.proc.stdin.close()
            except BrokenPipeError:
                traceback.print_exc()
            self.close()

    def read(self, bufsize=-1):
        """
        Return decompressor stdout data
        """
        if self.closed:
            raise OSError("writer closed")
        else:
            return self.read_io.read(bufsize)

    def close(self):
        """
        Wait for the external program to complete and do cleanup
        """
        if self.closed:
            return

        self.closed = True

        if self.proc is not None:
            self.thread.join()
            try:
                if self.proc.wait() != os.EX_OK:
                    if not self.killed:
                        writemsg(
                            colorize(
                                "BAD", "GPKG decompressor (external program) failed.\n"
                            )
                        )
                        raise CompressorOperationFailed("decompression failed")
            finally:
                self.proc.stdout.close()


class checksum_helper:
    """
    Do checksum generation, GnuPG signature generation, and verification
    """

    SIGNING = 0
    VERIFY = 1

    def __init__(self, settings, gpg_operation=None, detached=True, signature=None):
        """
        settings         # Portage settings
        gpg_operation    # either SIGNING or VERIFY
        signature        # GnuPG signature string used for GnuPG verification only
        """
        self.settings = settings
        self.gpg_operation = gpg_operation
        self.gpg_proc = None
        self.gpg_result = None
        self.gpg_output = None
        self.finished = False
        self.sign_file_path = None

        if (gpg_operation == checksum_helper.VERIFY) and (os.getuid() == 0):
            try:
                drop_user = self.settings.get("GPG_VERIFY_USER_DROP", "nobody")
                if drop_user == "":
                    self.uid = None
                else:
                    self.uid = pwd.getpwnam(drop_user).pw_uid
            except KeyError:
                writemsg(colorize("BAD", f"!!! Failed to find user {drop_user}.\n"))
                raise

            try:
                drop_group = self.settings.get("GPG_VERIFY_GROUP_DROP", "nogroup")
                if drop_group == "":
                    self.gid = None
                else:
                    self.gid = grp.getgrnam(drop_group).gr_gid
            except KeyError:
                writemsg(colorize("BAD", f"!!! Failed to find group {drop_group}.\n"))
                raise
        else:
            self.uid = None
            self.gid = None

        # Initialize the hash libs
        self.libs = {}
        for hash_name in MANIFEST2_HASH_DEFAULTS:
            self.libs[hash_name] = checksum.hashfunc_map[hash_name]._hashobject()

        # GnuPG
        env = self.settings.environ()
        if self.gpg_operation == checksum_helper.SIGNING:
            gpg_signing_base_command = self.settings.get(
                "BINPKG_GPG_SIGNING_BASE_COMMAND"
            )
            digest_algo = self.settings.get("BINPKG_GPG_SIGNING_DIGEST")
            gpg_home = self.settings.get("BINPKG_GPG_SIGNING_GPG_HOME")
            gpg_key = self.settings.get("BINPKG_GPG_SIGNING_KEY")

            if detached:
                gpg_detached = "--detach-sig"
            else:
                gpg_detached = "--clear-sign"

            if gpg_signing_base_command:
                gpg_signing_command = gpg_signing_base_command.replace(
                    "[PORTAGE_CONFIG]",
                    f"--homedir {gpg_home} "
                    f"--digest-algo {digest_algo} "
                    f"--local-user {gpg_key} "
                    f"{gpg_detached} "
                    "--batch --no-tty",
                )

                gpg_signing_command = shlex.split(
                    varexpand(gpg_signing_command, mydict=self.settings)
                )
                gpg_signing_command = [x for x in gpg_signing_command if x != ""]
                try:
                    env["GPG_TTY"] = os.ttyname(sys.stdout.fileno())
                except OSError:
                    pass
            else:
                raise CommandNotFound("GnuPG signing command is not set")

            self.gpg_proc = subprocess.Popen(
                gpg_signing_command,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                env=env,
            )

        elif self.gpg_operation == checksum_helper.VERIFY:
            if (signature is None) and (detached == True):
                raise MissingSignature("No signature provided")

            gpg_verify_base_command = self.settings.get(
                "BINPKG_GPG_VERIFY_BASE_COMMAND"
            )
            gpg_home = self.settings.get("BINPKG_GPG_VERIFY_GPG_HOME")

            if not gpg_verify_base_command:
                raise CommandNotFound("GnuPG verify command is not set")

            gpg_verify_command = gpg_verify_base_command.replace(
                "[PORTAGE_CONFIG]", f"--homedir {gpg_home} "
            )

            if detached:
                self.sign_file_fd, self.sign_file_path = tempfile.mkstemp(
                    ".sig", "portage-sign-"
                )

                gpg_verify_command = gpg_verify_command.replace(
                    "[SIGNATURE]", f"{self.sign_file_path} -"
                )

                # Create signature file and allow everyone to read
                with open(self.sign_file_fd, "wb") as sign:
                    sign.write(signature)
                os.chmod(self.sign_file_path, 0o644)
            else:
                gpg_verify_command = gpg_verify_command.replace(
                    "[SIGNATURE]", "--output - -"
                )

            gpg_verify_command = shlex.split(
                varexpand(gpg_verify_command, mydict=self.settings)
            )
            gpg_verify_command = [x for x in gpg_verify_command if x != ""]

            self.gpg_proc = subprocess.Popen(
                gpg_verify_command,
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                env=env,
                user=self.uid,
                group=self.gid,
            )

    def __del__(self):
        self.finish()

    def show_gpg_error(self, operation, gpg_error_lines):
        """
        Interpret GnuPG output to give a pretty error message
        with a summary if possible.
        """
        if operation == checksum_helper.VERIFY:
            operation_blurb = "verification failed"
        elif operation == checksum_helper.SIGNING:
            operation_blurb = "signing failed"

        # Attempt to give a nicer error the sniffing the status output.
        error_summaries = []
        portage_trust_helper = self.settings.get("PORTAGE_TRUST_HELPER", "")
        portage_trust_helper_msg = [
            # fmt: off
            "\t" + "Possible fix:",
            "\t" + f"Try running '{portage_trust_helper}', i.e., $PORTAGE_TRUST_HELPER, as root",
            # fmt: on
        ]

        def _match_list(regex: re.Pattern, msgs: list) -> list[re.Match]:
            return list(filter(lambda s: re.match(regex, s), msgs))

        errors = 0
        if _match_list(r"^\[GNUPG:\] NODATA", gpg_error_lines):
            errors += 1
            error_summaries.append("binpkg appears unsigned (missing any signature)")
        if _match_list(r"^\[GNUPG:\] NO_PUBKEY", gpg_error_lines):
            errors += 1
            error_summaries.append("binpkg signed with at least one unknown key.")
            error_summaries.extend(portage_trust_helper_msg)
        if _match_list(r"^\[GNUPG:\] TRUST_UNDEFINED", gpg_error_lines):
            errors += 1
            error_summaries.append("binpkg signed with a known key of undefined trust.")
            error_summaries.extend(portage_trust_helper_msg)

        # Don't show any summary if it's ambiguous, in case of
        # a malformed signature.
        if errors != 1:
            error_summaries = ["(none available)"]

        out = portage.output.EOutput()
        msg = [f"Binary package is not usable ({operation_blurb}):"]
        msg.append(" Summary:")
        msg.extend("\t" + line for line in error_summaries)
        msg.append("")
        msg.append(" Raw GnuPG output:")
        msg.extend("\t" + line for line in gpg_error_lines)
        [out.eerror(line) for line in msg]

    def _check_gpg_status(self, gpg_status: bytes) -> None:
        """
        Check GnuPG status log for extra info.
        GnuPG will return OK even if the signature owner is not trusted.
        """
        good_signature = False
        trust_signature = False

        for l in gpg_status.splitlines():
            if l.startswith(b"[GNUPG:] GOODSIG"):
                good_signature = True

            if l.startswith(b"[GNUPG:] TRUST_ULTIMATE") or l.startswith(
                b"[GNUPG:] TRUST_FULLY"
            ):
                trust_signature = True

        if (not good_signature) or (not trust_signature):
            gpg_error_lines = self.gpg_result.decode(
                "UTF-8", errors="replace"
            ).splitlines()
            self.show_gpg_error(checksum_helper.VERIFY, gpg_error_lines)
            raise InvalidSignature("GnuPG verification failed")

    def update(self, data):
        """
        Write data to both hash libs and GnuPG stdin.
        """
        for c in self.libs:
            self.libs[c].update(data)

        if self.gpg_proc is not None:
            self.gpg_proc.stdin.write(data)

    def finish(self):
        """
        Tell GnuPG that the file is EOF, then get results, then cleanup.
        """
        if self.finished or self.gpg_proc is None:
            return

        # Tell GnuPG EOF
        self.gpg_proc.stdin.close()

        return_code = self.gpg_proc.wait()

        if self.sign_file_path:
            os.remove(self.sign_file_path)

        self.finished = True
        self.gpg_result = self.gpg_proc.stderr.read()
        self.gpg_output = self.gpg_proc.stdout.read()
        self.gpg_proc.stdout.close()
        self.gpg_proc.stderr.close()

        if return_code == os.EX_OK:
            if self.gpg_operation == checksum_helper.VERIFY:
                # os.EX_OK isn't a guarantee of everything being fine,
                # so do more checks with `status-fd` output.
                self._check_gpg_status(self.gpg_result)
            return

        gpg_error_lines = self.gpg_result.decode("UTF-8", errors="replace").splitlines()

        if self.gpg_operation == checksum_helper.SIGNING:
            self.show_gpg_error(checksum_helper.SIGNING, gpg_error_lines)
            raise GPGException("GnuPG signing failed")
        elif self.gpg_operation == checksum_helper.VERIFY:
            self.show_gpg_error(checksum_helper.VERIFY, gpg_error_lines)
            raise InvalidSignature("GnuPG verification failed")


class tar_safe_extract:
    """
    A safer version of TarFile's extractall that performs a sanity check.
    Note that this does not solve all security problems.
    """

    def __init__(self, tar: tarfile.TarFile, prefix: str = ""):
        """
        tar: an opened TarFile that ready to be read.
        prefix: a optional prefix for an inner directory should be considered
        as the root directory. e.g. "metadata" and "image".
        """
        self.tar = tar
        self.prefix = prefix
        self.closed = False
        self.file_list = []

    def extractall(self, dest_dir: str):
        """
        Extract all files to a temporary directory in the dest_dir, and move
        them to the dest_dir after sanity check.
        """
        if self.closed:
            raise OSError("Tar file is closed.")
        temp_dir = tempfile.TemporaryDirectory(dir=dest_dir)
        # The below tar member security checks can be refactored as a filter function
        # that raises an exception. Use tarfile.fully_trusted_filter for now, which
        # is simply an identity function:
        # def fully_trusted_filter(member, dest_path):
        #     return member
        try:
            self.tar.extraction_filter = tarfile.fully_trusted_filter
        except AttributeError:
            pass
        try:
            while True:
                member = self.tar.next()
                if member is None:
                    break
                if (member.name in self.file_list) or (
                    os.path.join(".", member.name) in self.file_list
                ):
                    writemsg(
                        colorize(
                            "BAD", f"Danger: duplicate files detected: {member.name}\n"
                        )
                    )
                    raise ValueError("Duplicate files detected.")
                if member.name.startswith("/"):
                    writemsg(
                        colorize(
                            "BAD", f"Danger: absolute path detected: {member.name}\n"
                        )
                    )
                    raise ValueError("Absolute path detected.")
                if member.name.startswith("../") or ("/../" in member.name):
                    writemsg(
                        colorize(
                            "BAD", f"Danger: path traversal detected: {member.name}\n"
                        )
                    )
                    raise ValueError("Path traversal detected.")
                if member.isdev():
                    writemsg(
                        colorize(
                            "BAD", f"Danger: device file detected: {member.name}\n"
                        )
                    )
                    raise ValueError("Device file detected.")
                if member.islnk() and (member.linkname not in self.file_list):
                    writemsg(
                        colorize(
                            "BAD", f"Danger: hardlink escape detected: {member.name}\n"
                        )
                    )
                    raise ValueError("Hardlink escape detected.")

                self.file_list.append(member.name)
                self.tar.extract(member, path=temp_dir.name)

            data_dir = os.path.join(temp_dir.name, self.prefix)
            for file in os.listdir(data_dir):
                shutil.move(os.path.join(data_dir, file), os.path.join(dest_dir, file))
        finally:
            temp_dir.cleanup()
            self.closed = True


class gpkg:
    """
    Gentoo binary package
    https://www.gentoo.org/glep/glep-0078.html
    """

    def __init__(self, settings, basename=None, gpkg_file=None, verify_signature=None):
        """
        gpkg class handles all gpkg operations for one package.
        basename is the package basename.
        gpkg_file should exist as a file path for reads or will be created.
        """
        self.settings = settings
        self.gpkg_version = "gpkg-1"
        if gpkg_file is None:
            self.gpkg_file = None
        else:
            self.gpkg_file = _unicode_decode(
                gpkg_file, encoding=_encodings["fs"], errors="strict"
            )

        if basename is None:
            self.basename = None
        else:
            self.basename = basename.split("/", maxsplit=1)[-1]

        self.checksums = []
        self.manifest_old = []
        self.signature_exist = None
        self.prefix = None

        # Compression is the compression algorithm. No compression
        # if set to None.
        self.compression = self.settings.get("BINPKG_COMPRESS", None)
        if self.compression in ["", "none"]:
            self.compression = None

        # Whether to sign the package or not
        self.create_signature = "binpkg-signing" in self.settings.features

        # If `verify-signature` is unset in binrepos.conf, use the FEATURES
        # flags instead.
        if verify_signature is None:
            # request_signature is whether signature files are mandatory.
            # If true, any missing signature file will cause processing to be
            # rejected.
            self.request_signature = (
                "binpkg-request-signature" in self.settings.features
            )

            # verify_signature is whether to verify package signatures or not.
            # In rare cases, the user may want to ignore signature, e.g.
            # a package with an expired signature.
            self.verify_signature = (
                "binpkg-ignore-signature" not in self.settings.features
            )
        else:
            self.verify_signature = verify_signature
            self.request_signature = verify_signature

        # FEATURES should override in one direction if they're stronger
        # and explicitly set. This also makes testing easier.
        if "binpkg-request-signature" in self.settings.features:
            self.request_signature = True
            self.verify_signature = True
        elif "binpkg-ignore-signature" in self.settings.features:
            self.request_signature = False
            self.verify_signature = False

        self.ext_list = {
            "gzip": ".gz",
            "bzip2": ".bz2",
            "lz4": ".lz4",
            "lzip": ".lz",
            "lzop": ".lzo",
            "xz": ".xz",
            "zstd": ".zst",
        }

    @staticmethod
    def _strip_metadata_prefix(path):
        prefix = "metadata/"
        if not path.startswith(prefix):
            raise InvalidBinaryPackageFormat(f"Invalid metadata path: {path}")
        return path[len(prefix) :]

    def unpack_metadata(self, dest_dir=None):
        """
        Unpack metadata to dest_dir.
        If dest_dir is None, return files and values in dict.
        The dict key will be UTF-8, not bytes.
        """
        self._verify_binpkg(metadata_only=True)

        with tarfile.open(self.gpkg_file, "r") as container:
            metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
                container, "metadata"
            )

            with tar_stream_reader(
                container.extractfile(metadata_tarinfo),
                self._get_decompression_cmd(metadata_comp),
            ) as metadata_reader:
                metadata_tar = io.BytesIO(metadata_reader.read())

            with tarfile.open(mode="r:", fileobj=metadata_tar) as metadata:
                if dest_dir is None:
                    metadata_ = {
                        self._strip_metadata_prefix(k.name): metadata.extractfile(
                            k
                        ).read()
                        for k in metadata.getmembers()
                    }
                else:
                    metadata_safe = tar_safe_extract(metadata, "metadata")
                    metadata_safe.extractall(dest_dir)
                    metadata_ = True
            metadata_tar.close()
        return metadata_

    def get_metadata(self, want=None):
        """
        Get package metadata.
        if want is list, return all want key-values in dict
        if want is str, return the want key value
        """
        if want is None:
            return self.unpack_metadata()
        elif isinstance(want, str):
            metadata = self.unpack_metadata()
            metadata_want = metadata.get(want, None)
            return metadata_want
        else:
            metadata = self.unpack_metadata()
            metadata_want = {k: metadata.get(k, None) for k in want}
            return metadata_want

    def get_metadata_url(self, url, want=None):
        """
        Return the requested metadata from url gpkg.
        Default return all metadata.
        Use 'want' to get specific name from metadata.
        This method only support the correct package format.
        Wrong files order or incorrect basename will be considered invalid
        to reduce potential attacks.
        Signatures will only be checked if the signature file is the next file.
        Manifest will be ignored since it will be at the end of package.
        """
        # The init download file head size
        init_size = 51200

        # Load remote container
        container_file = io.BytesIO(
            urlopen(url, headers={"Range": "bytes=0-" + str(init_size)}).read()
        )

        # Check gpkg and metadata
        with tarfile.open(mode="r", fileobj=container_file) as container:
            if self.gpkg_version not in (
                os.path.basename(f) for f in container.getnames()
            ):
                raise InvalidBinaryPackageFormat("Invalid gpkg file.")

            metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
                container, "metadata"
            )

            # Extra 10240 bytes for signature
            end_size = metadata_tarinfo.offset_data + metadata_tarinfo.size + 10240
            _, remainder = divmod(end_size, 512)
            end_size += 512 - remainder

            # If need more data
            if end_size > 10000000:
                raise InvalidBinaryPackageFormat("metadata too large " + str(end_size))
            if end_size > init_size:
                container_file.seek(0, io.SEEK_END)
                container_file.write(
                    urlopen(
                        url,
                        headers={"Range": f"bytes={init_size + 1}-{end_size}"},
                    ).read()
                )

        container_file.seek(0)

        # Reload and process full metadata
        with tarfile.open(mode="r", fileobj=container_file) as container:
            metadata_tarinfo, metadata_comp = self._get_inner_tarinfo(
                container, "metadata"
            )

            # Verify metadata file signature if needed
            # (binpkg-ignore-signature can override this)
            signature_filename = metadata_tarinfo.name + ".sig"
            if signature_filename in container.getnames():
                if self.request_signature and self.verify_signature:
                    metadata_signature = container.extractfile(
                        signature_filename
                    ).read()
                    checksum_info = checksum_helper(
                        self.settings,
                        gpg_operation=checksum_helper.VERIFY,
                        signature=metadata_signature,
                    )
                    checksum_info.update(container.extractfile(metadata_tarinfo).read())
                    checksum_info.finish()

            # Load metadata
            with tar_stream_reader(
                container.extractfile(metadata_tarinfo),
                self._get_decompression_cmd(metadata_comp),
            ) as metadata_reader:
                metadata_file = io.BytesIO(metadata_reader.read())

            with tarfile.open(mode="r:", fileobj=metadata_file) as metadata:
                if want is None:
                    metadata_ = {
                        self._strip_metadata_prefix(k.name): metadata.extractfile(
                            k
                        ).read()
                        for k in metadata.getmembers()
                    }
                else:
                    metadata_ = {
                        self._strip_metadata_prefix(k.name): metadata.extractfile(
                            k
                        ).read()
                        for k in metadata.getmembers()
                        if k in want
                    }
            metadata_file.close()
        container_file.close()
        return metadata_

    def compress(self, root_dir, metadata, clean=False):
        """
        Use initialized configuation create new gpkg file from root_dir.
        Will overwrite any exists file.
        metadata is a dict, the key will be file name, the value will be
        the file contents.
        """

        root_dir = normalize_path(
            _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
        )

        # Get pre image info
        container_tar_format, image_tar_format = self._get_tar_format_from_stats(
            *self._check_pre_image_files(root_dir)
        )

        # Long CPV
        if len(self.basename) >= 154:
            container_tar_format = tarfile.GNU_FORMAT

        # gpkg container
        container = tarfile.TarFile(
            name=self.gpkg_file, mode="w", format=container_tar_format
        )

        # gpkg version
        gpkg_version_file = tarfile.TarInfo(
            os.path.join(self.basename, self.gpkg_version)
        )
        gpkg_version_file.mtime = datetime.now().timestamp()
        container.addfile(gpkg_version_file)
        checksum_info = checksum_helper(self.settings)
        checksum_info.finish()
        self._record_checksum(checksum_info, gpkg_version_file)

        compression_cmd = self._get_compression_cmd()

        # metadata
        self._add_metadata(container, metadata, compression_cmd)

        # image
        if self.create_signature:
            checksum_info = checksum_helper(
                self.settings, gpg_operation=checksum_helper.SIGNING
            )
        else:
            checksum_info = checksum_helper(self.settings)

        image_tarinfo = self._create_tarinfo("image")
        image_tarinfo.mtime = datetime.now().timestamp()
        with tar_stream_writer(
            image_tarinfo, container, image_tar_format, compression_cmd, checksum_info
        ) as image_writer:
            with tarfile.open(
                mode="w|", fileobj=image_writer, format=image_tar_format
            ) as image_tar:
                image_tar.add(root_dir, "image", recursive=True)

        image_tarinfo = container.getmember(image_tarinfo.name)
        self._record_checksum(checksum_info, image_tarinfo)

        if self.create_signature:
            self._add_signature(checksum_info, image_tarinfo, container)

        self._add_manifest(container)

        # Check if all directories are the same in the container
        prefix = os.path.commonpath(container.getnames())
        if not prefix:
            raise InvalidBinaryPackageFormat(
                f"gpkg file structure mismatch in {self.gpkg_file}"
            )

        container.close()

    def decompress(self, decompress_dir):
        """
        Decompress current gpkg to decompress_dir
        """
        decompress_dir = normalize_path(
            _unicode_decode(decompress_dir, encoding=_encodings["fs"], errors="strict")
        )

        self._verify_binpkg()
        os.makedirs(decompress_dir, mode=0o755, exist_ok=True)

        with tarfile.open(self.gpkg_file, "r") as container:
            image_tarinfo, image_comp = self._get_inner_tarinfo(container, "image")

            with tar_stream_reader(
                container.extractfile(image_tarinfo),
                self._get_decompression_cmd(image_comp),
            ) as image_tar:
                with tarfile.open(mode="r|", fileobj=image_tar) as image:
                    try:
                        image_safe = tar_safe_extract(image, "image")
                        image_safe.extractall(decompress_dir)
                        image_tar.close()
                    except Exception as ex:
                        writemsg(colorize("BAD", "!!!Extract failed.\n"))
                        raise
                    finally:
                        if not image_tar.closed:
                            image_tar.kill()

    def update_metadata(self, metadata, new_basename=None, force=False):
        """
        Update metadata in the gpkg file.
        """
        self._verify_binpkg()
        self.checksums = []
        if self.signature_exist and not force:
            raise SignedPackage("Cannot update a signed gpkg file")

        if new_basename is None:
            if self.basename:
                new_basename = self.basename
            elif self.prefix:
                new_basename = self.prefix
            else:
                raise InvalidBinaryPackageFormat("No basename or prefix specified")
        else:
            new_basename = new_basename.split("/", maxsplit=1)[-1]
            self.basename = new_basename

        with open(self.gpkg_file, "rb") as container:
            container_tar_format = self._get_tar_format(container)
            if container_tar_format is None:
                raise InvalidBinaryPackageFormat("Cannot identify tar format")

        # container
        with (
            tempfile.NamedTemporaryFile(
                dir=os.path.dirname(self.gpkg_file),
                prefix=f"{os.path.basename(self.gpkg_file)}.{portage.getpid()}",
                delete=False,
            ) as tmp_gpkg_file,
            tarfile.TarFile(
                name=tmp_gpkg_file.name, mode="w", format=container_tar_format
            ) as container,
        ):
            os.fchmod(tmp_gpkg_file.fileno(), 0o644)
            tmp_gpkg_file_name = tmp_gpkg_file.name
            tmp_gpkg_file.close()

            # gpkg version
            gpkg_version_file = tarfile.TarInfo(
                os.path.join(new_basename, self.gpkg_version)
            )
            gpkg_version_file.mtime = datetime.now().timestamp()
            container.addfile(gpkg_version_file)
            checksum_info = checksum_helper(self.settings)
            checksum_info.finish()
            self._record_checksum(checksum_info, gpkg_version_file)

            compression_cmd = self._get_compression_cmd()

            # metadata
            self._add_metadata(container, metadata, compression_cmd)

            # reuse other stuffs
            with tarfile.open(self.gpkg_file, "r") as container_old:
                manifest_old = self.manifest_old.copy()

                for m in manifest_old:
                    file_name_old = m[1]
                    if os.path.basename(file_name_old) == self.gpkg_version:
                        continue
                    if os.path.basename(file_name_old).startswith("metadata"):
                        continue
                    old_data_tarinfo = container_old.getmember(
                        os.path.join(self.prefix, file_name_old)
                    )
                    new_data_tarinfo = copy(old_data_tarinfo)
                    new_file_path = list(os.path.split(new_data_tarinfo.name))
                    new_file_path[0] = new_basename
                    new_data_tarinfo.name = os.path.join(*new_file_path)
                    container.addfile(
                        new_data_tarinfo, container_old.extractfile(old_data_tarinfo)
                    )
                    self.checksums.append(m)

            self._add_manifest(container)

            # Check if all directories are the same in the container
            prefix = os.path.commonpath(container.getnames())
            if not prefix:
                raise InvalidBinaryPackageFormat(
                    f"gpkg file structure mismatch in {self.gpkg_file}; files: "
                    f"{container.getnames()}"
                )

        shutil.move(tmp_gpkg_file_name, self.gpkg_file)

    def update_signature(self, keep_current_signature=False):
        """
        Add / update signature in the gpkg file.
        if keep_current_signature is True, keep the current signature, otherwise re-sign it.
        """
        self.create_signature = True
        self._verify_binpkg()
        self.checksums = []

        with open(self.gpkg_file, "rb") as container:
            container_tar_format = self._get_tar_format(container)
            if container_tar_format is None:
                raise InvalidBinaryPackageFormat("Cannot identify tar format")

        # container
        with (
            tempfile.NamedTemporaryFile(
                dir=os.path.dirname(self.gpkg_file),
                prefix=f"{os.path.basename(self.gpkg_file)}.{portage.getpid()}",
                delete=False,
            ) as tmp_gpkg_file,
            tarfile.TarFile(
                name=tmp_gpkg_file.name, mode="w", format=container_tar_format
            ) as container,
        ):
            os.fchmod(tmp_gpkg_file.fileno(), 0o644)
            tmp_gpkg_file_name = tmp_gpkg_file.name
            tmp_gpkg_file.close()

            # gpkg version
            gpkg_version_file = tarfile.TarInfo(
                os.path.join(self.prefix, self.gpkg_version)
            )
            gpkg_version_file.mtime = datetime.now().timestamp()
            container.addfile(gpkg_version_file)
            checksum_info = checksum_helper(self.settings)
            checksum_info.finish()
            self._record_checksum(checksum_info, gpkg_version_file)

            # reuse other stuffs
            with tarfile.open(self.gpkg_file, "r") as container_old:
                manifest_old = self.manifest_old.copy()
                file_list_old = [f[1] for f in manifest_old]

                for m in manifest_old:
                    file_name_old = m[1]
                    if os.path.basename(file_name_old) == self.gpkg_version:
                        continue
                    if os.path.basename(file_name_old).endswith(".sig"):
                        continue
                    old_data_tarinfo = container_old.getmember(
                        os.path.join(self.prefix, file_name_old)
                    )
                    new_data_tarinfo = copy(old_data_tarinfo)

                    container.addfile(
                        new_data_tarinfo, container_old.extractfile(old_data_tarinfo)
                    )
                    self.checksums.append(m)

                    # Check if signature file exists and reuse or create new one.
                    if keep_current_signature and (
                        file_name_old + ".sig" in file_list_old
                    ):
                        old_data_sign_tarinfo = container_old.getmember(
                            file_name_old + ".sig"
                        )
                        new_data_sign_tarinfo = copy(old_data_sign_tarinfo)
                        container.addfile(
                            new_data_sign_tarinfo,
                            container_old.extractfile(old_data_sign_tarinfo),
                        )
                        for manifest_sign in manifest_old:
                            if manifest_sign[1] == file_name_old + ".sig":
                                self.checksums.append(manifest_sign)
                                break
                    else:
                        checksum_info = checksum_helper(
                            self.settings, gpg_operation=checksum_helper.SIGNING
                        )
                        checksum_info.update(
                            container_old.extractfile(old_data_tarinfo).read()
                        )
                        checksum_info.finish()
                        self._add_signature(checksum_info, new_data_tarinfo, container)

            self._add_manifest(container)

            # Check if all directories are the same in the container
            prefix = os.path.commonpath(container.getnames())
            if not prefix:
                raise InvalidBinaryPackageFormat(
                    f"gpkg file structure mismatch in {self.gpkg_file}"
                )

        shutil.move(tmp_gpkg_file_name, self.gpkg_file)

    def _add_metadata(self, container, metadata, compression_cmd):
        """
        Add metadata to container
        """
        if metadata is None:
            metadata = {}
        metadata_tarinfo = self._create_tarinfo("metadata")
        metadata_tarinfo.mtime = datetime.now().timestamp()

        if self.create_signature:
            checksum_info = checksum_helper(
                self.settings, gpg_operation=checksum_helper.SIGNING
            )
        else:
            checksum_info = checksum_helper(self.settings)

        with tar_stream_writer(
            metadata_tarinfo,
            container,
            tarfile.USTAR_FORMAT,
            compression_cmd,
            checksum_info,
        ) as metadata_writer:
            with tarfile.open(
                mode="w|", fileobj=metadata_writer, format=tarfile.USTAR_FORMAT
            ) as metadata_tar:
                for m in metadata:
                    m_info = tarfile.TarInfo(os.path.join("metadata", m))
                    m_info.mtime = datetime.now().timestamp()

                    if isinstance(metadata[m], bytes):
                        m_data = io.BytesIO(metadata[m])
                    else:
                        m_data = io.BytesIO(metadata[m].encode("UTF-8"))

                    m_data.seek(0, io.SEEK_END)
                    m_info.size = m_data.tell()
                    m_data.seek(0)
                    metadata_tar.addfile(m_info, m_data)
                    m_data.close()

        metadata_tarinfo = container.getmember(metadata_tarinfo.name)
        self._record_checksum(checksum_info, metadata_tarinfo)

        if self.create_signature:
            self._add_signature(checksum_info, metadata_tarinfo, container)

    def _quickpkg(self, contents, metadata, root_dir, protect=None):
        """
        Similar to compress, but for quickpkg.
        Will compress the given files to image with root,
        ignoring all other files.
        """
        eout = EOutput()

        protect_file = io.BytesIO(
            b"# empty file because --include-config=n when `quickpkg` was used\n"
        )
        protect_file.seek(0, io.SEEK_END)
        protect_file_size = protect_file.tell()

        root_dir = normalize_path(
            _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
        )

        # Get pre image info
        container_tar_format, image_tar_format = self._get_tar_format_from_stats(
            *self._check_pre_quickpkg_files(contents, root_dir, ignore_missing=True)
        )

        # Long CPV
        if len(self.basename) >= 154:
            container_tar_format = tarfile.GNU_FORMAT

        # GPKG container
        container = tarfile.TarFile(
            name=self.gpkg_file, mode="w", format=container_tar_format
        )

        # GPKG version
        gpkg_version_file = tarfile.TarInfo(
            os.path.join(self.basename, self.gpkg_version)
        )
        gpkg_version_file.mtime = datetime.now().timestamp()
        container.addfile(gpkg_version_file)
        checksum_info = checksum_helper(self.settings)
        checksum_info.finish()
        self._record_checksum(checksum_info, gpkg_version_file)

        compression_cmd = self._get_compression_cmd()
        # Metadata
        self._add_metadata(container, metadata, compression_cmd)

        # Image
        if self.create_signature:
            checksum_info = checksum_helper(
                self.settings, gpg_operation=checksum_helper.SIGNING
            )
        else:
            checksum_info = checksum_helper(self.settings)

        paths = list(contents)
        paths.sort()
        image_tarinfo = self._create_tarinfo("image")
        image_tarinfo.mtime = datetime.now().timestamp()
        with tar_stream_writer(
            image_tarinfo, container, image_tar_format, compression_cmd, checksum_info
        ) as image_writer:
            with tarfile.open(
                mode="w|", fileobj=image_writer, format=image_tar_format
            ) as image_tar:
                if len(paths) == 0:
                    tarinfo = image_tar.tarinfo("image")
                    tarinfo.type = tarfile.DIRTYPE
                    tarinfo.size = 0
                    tarinfo.mode = 0o755
                    image_tar.addfile(tarinfo)

                for path in paths:
                    try:
                        lst = os.lstat(path)
                    except OSError as e:
                        if e.errno != errno.ENOENT:
                            raise
                        eout.ewarn(f'Missing file from local system: "{path}"')
                        del e
                        continue
                    contents_type = contents[path][0]
                    if path.startswith(root_dir):
                        arcname = "image/" + path[len(root_dir) :]
                    else:
                        raise ValueError(f"invalid root argument: '{root_dir}'")
                    live_path = path
                    if (
                        "dir" == contents_type
                        and not stat.S_ISDIR(lst.st_mode)
                        and os.path.isdir(live_path)
                    ):
                        # Even though this was a directory in the original ${D}, it exists
                        # as a symlink to a directory in the live filesystem.  It must be
                        # recorded as a real directory in the tar file to ensure that tar
                        # can properly extract it's children.
                        live_path = os.path.realpath(live_path)
                        lst = os.lstat(live_path)

                    # Since os.lstat() inside TarFile.gettarinfo() can trigger a
                    # UnicodeEncodeError when python has something other than utf_8
                    # return from sys.getfilesystemencoding() (as in bug #388773),
                    # we implement the needed functionality here, using the result
                    # of our successful lstat call. An alternative to this would be
                    # to pass in the fileobj argument to TarFile.gettarinfo(), so
                    # that it could use fstat instead of lstat. However, that would
                    # have the unwanted effect of dereferencing symlinks.

                    tarinfo = image_tar.tarinfo(arcname)
                    tarinfo.mode = lst.st_mode
                    tarinfo.uid = lst.st_uid
                    tarinfo.gid = lst.st_gid
                    tarinfo.size = 0
                    tarinfo.mtime = lst.st_mtime
                    tarinfo.linkname = ""
                    if stat.S_ISREG(lst.st_mode):
                        inode = (lst.st_ino, lst.st_dev)
                        if (
                            lst.st_nlink > 1
                            and inode in image_tar.inodes
                            and arcname != image_tar.inodes[inode]
                        ):
                            tarinfo.type = tarfile.LNKTYPE
                            tarinfo.linkname = image_tar.inodes[inode]
                        else:
                            image_tar.inodes[inode] = arcname
                            tarinfo.type = tarfile.REGTYPE
                            tarinfo.size = lst.st_size
                    elif stat.S_ISDIR(lst.st_mode):
                        tarinfo.type = tarfile.DIRTYPE
                    elif stat.S_ISLNK(lst.st_mode):
                        tarinfo.type = tarfile.SYMTYPE
                        tarinfo.linkname = os.readlink(live_path)
                    else:
                        continue
                    try:
                        tarinfo.uname = pwd.getpwuid(tarinfo.uid)[0]
                    except KeyError:
                        pass
                    try:
                        tarinfo.gname = grp.getgrgid(tarinfo.gid)[0]
                    except KeyError:
                        pass

                    if stat.S_ISREG(lst.st_mode):
                        if protect and protect(path):
                            protect_file.seek(0)
                            tarinfo.size = protect_file_size
                            image_tar.addfile(tarinfo, protect_file)
                        else:
                            path_bytes = _unicode_encode(
                                path, encoding=_encodings["fs"], errors="strict"
                            )

                            with open(path_bytes, "rb") as f:
                                image_tar.addfile(tarinfo, f)

                    else:
                        image_tar.addfile(tarinfo)

        image_tarinfo = container.getmember(image_tarinfo.name)
        self._record_checksum(checksum_info, image_tarinfo)

        if self.create_signature:
            self._add_signature(checksum_info, image_tarinfo, container)

        self._add_manifest(container)

        # Check if all directories are the same in the container
        prefix = os.path.commonpath(container.getnames())
        if not prefix:
            raise InvalidBinaryPackageFormat(
                f"gpkg file structure mismatch in {self.gpkg_file}"
            )

        container.close()

    def _record_checksum(self, checksum_info, tarinfo):
        """
        Record checksum result for the given file.
        Replace old checksum if it already exists.
        """

        # Remove prefix directory from the filename
        file_name = os.path.basename(tarinfo.name)

        for c in self.checksums:
            if c[1] == file_name:
                self.checksums.remove(c)

        checksum_record = ["DATA", file_name, str(tarinfo.size)]

        for c in checksum_info.libs:
            checksum_record.append(c)
            checksum_record.append(checksum_info.libs[c].hexdigest())

        self.checksums.append(checksum_record)

    def _add_manifest(self, container):
        """
        Add Manifest to the container based on current checksums.
        Create GnuPG signature if needed.
        """
        manifest = io.BytesIO()

        for m in self.checksums:
            manifest.write((" ".join(m) + "\n").encode("UTF-8"))

        if self.create_signature:
            checksum_info = checksum_helper(
                self.settings, gpg_operation=checksum_helper.SIGNING, detached=False
            )
            checksum_info.update(manifest.getvalue())
            checksum_info.finish()
            manifest.seek(0)
            manifest.write(checksum_info.gpg_output)

        if self.basename is not None:
            basename = self.basename
        elif self.prefix is not None:
            basename = self.prefix
        else:
            raise InvalidBinaryPackageFormat("No basename or prefix specified")

        manifest_tarinfo = tarfile.TarInfo(os.path.join(basename, "Manifest"))
        manifest_tarinfo.size = manifest.tell()
        manifest_tarinfo.mtime = datetime.now().timestamp()
        manifest.seek(0)
        container.addfile(manifest_tarinfo, manifest)
        manifest.close()

    def _load_manifest(self, manifest_string):
        """
        Check, load, and return manifest in a list by files
        """
        manifest = []
        manifest_filenames = []

        for manifest_record in manifest_string.splitlines():
            if manifest_record == "":
                continue
            manifest_record = manifest_record.strip().split()

            if manifest_record[0] != "DATA":
                raise DigestException("invalied Manifest")

            if manifest_record[1] in manifest_filenames:
                raise DigestException("Manifest duplicate file exists")

            try:
                int(manifest_record[2])
            except ValueError:
                raise DigestException("Manifest invalied file size")

            manifest.append(manifest_record)
            manifest_filenames.append(manifest_record[1])

        return manifest

    def _add_signature(self, checksum_info, tarinfo, container, manifest=True):
        """
        Add GnuPG signature for the given tarinfo file.
        manifest: add to manifest
        """
        if checksum_info.gpg_output is None:
            raise GPGException("GnuPG signature does not exist")

        signature = io.BytesIO(checksum_info.gpg_output)
        signature_tarinfo = tarfile.TarInfo(f"{tarinfo.name}.sig")
        signature_tarinfo.size = len(signature.getvalue())
        signature_tarinfo.mtime = datetime.now().timestamp()
        container.addfile(signature_tarinfo, signature)

        if manifest:
            signature_checksum_info = checksum_helper(self.settings)
            signature.seek(0)
            signature_checksum_info.update(signature.read())
            signature_checksum_info.finish()
            self._record_checksum(signature_checksum_info, signature_tarinfo)

        signature.close()

    def _verify_binpkg(self, metadata_only=False):
        """
        Verify current GPKG file.
        """
        # Check file path
        if self.gpkg_file is None:
            raise FileNotFound("no gpkg file provided")

        # Check if is file
        if not os.path.isfile(self.gpkg_file):
            raise FileNotFound(f"File not found {self.gpkg_file}")

        # Check if is tar file
        with open(self.gpkg_file, "rb") as container:
            container_tar_format = self._get_tar_format(container)
            if container_tar_format is None:
                get_binpkg_format(self.gpkg_file, check_file=True)
                raise InvalidBinaryPackageFormat(
                    f"Cannot identify tar format: {self.gpkg_file}"
                )

        # Check container
        with tarfile.open(self.gpkg_file, "r") as container:
            try:
                container_files = container.getnames()
            except tarfile.ReadError:
                get_binpkg_format(self.gpkg_file, check_file=True)
                raise InvalidBinaryPackageFormat(
                    f"Cannot read tar file: {self.gpkg_file}"
                )

            # Check if gpkg version file exists in any place
            if self.gpkg_version not in (os.path.basename(f) for f in container_files):
                get_binpkg_format(self.gpkg_file, check_file=True)
                raise InvalidBinaryPackageFormat(f"Invalid gpkg file: {self.gpkg_file}")

            # Check how many layers are in the container
            for f in container_files:
                if f.startswith("/"):
                    raise InvalidBinaryPackageFormat(
                        f"gpkg file structure mismatch '{f}' in {self.gpkg_file}"
                    )
                if f.count("/") != 1:
                    raise InvalidBinaryPackageFormat(
                        f"gpkg file structure mismatch '{f}' in {self.gpkg_file}"
                    )

            # Check if all directories are the same in the container
            prefix = os.path.commonpath(container_files)
            if not prefix:
                raise InvalidBinaryPackageFormat(
                    f"gpkg file structure mismatch in {self.gpkg_file}, {str(container_files)}"
                )

            gpkg_version_file = os.path.join(prefix, self.gpkg_version)

            # If any signature exists, we assume all files have signature.
            if any(f.endswith(".sig") for f in container_files):
                signature_exist = True
            else:
                signature_exist = False

            # Check if all files are unique to avoid same name attack
            container_files_unique = []
            for f in container_files:
                if f in container_files_unique:
                    raise InvalidBinaryPackageFormat(
                        f"Duplicate file {f} exist, potential attack?"
                    )
                container_files_unique.append(f)

            del container_files_unique

            # Add all files to check list
            unverified_files = container_files.copy()

            # Check Manifest file
            manifest_filename = os.path.join(prefix, "Manifest")
            if manifest_filename not in unverified_files:
                raise MissingSignature(f"Manifest not found: {self.gpkg_file}")

            manifest_file = container.extractfile(manifest_filename)
            manifest_data = manifest_file.read()
            manifest_file.close()

            if b"-----BEGIN PGP SIGNATURE-----" in manifest_data:
                signature_exist = True

            # Check Manifest signature if needed.
            # (binpkg-ignore-signature can override this.)
            if self.request_signature or signature_exist:
                checksum_info = checksum_helper(
                    self.settings, gpg_operation=checksum_helper.VERIFY, detached=False
                )

                try:
                    checksum_info.update(manifest_data)
                    checksum_info.finish()
                except (InvalidSignature, MissingSignature):
                    if self.verify_signature:
                        raise

                manifest_data = checksum_info.gpg_output
                unverified_files.remove(manifest_filename)
            else:
                unverified_files.remove(manifest_filename)

            # Load manifest and create manifest check list
            manifest = self._load_manifest(manifest_data.decode("UTF-8"))
            unverified_manifest = manifest.copy()

            # Check all remaining files
            for f in unverified_files.copy():
                if f.endswith(".sig"):
                    f_signature = None
                else:
                    f_signature = f + ".sig"

                # Find current file manifest record
                manifest_record = None
                for m in manifest:
                    if m[1] == os.path.basename(f):
                        manifest_record = m

                if manifest_record is None:
                    raise DigestException(f"{f} checksum not found in {self.gpkg_file}")

                if int(manifest_record[2]) != int(container.getmember(f).size):
                    raise DigestException(
                        f"{f} file size mismatched in {self.gpkg_file}"
                    )

                # Ignore image file and signature if not needed
                if os.path.basename(f).startswith("image") and metadata_only:
                    unverified_files.remove(f)
                    unverified_manifest.remove(manifest_record)
                    continue

                # Verify current file signature if needed
                # (binpkg-ignore-signature can override this.)
                if (
                    (self.request_signature or signature_exist)
                    and self.verify_signature
                    and f_signature
                ):
                    if f_signature in unverified_files:
                        signature_file = container.extractfile(f_signature)
                        signature = signature_file.read()
                        signature_file.close()
                        checksum_info = checksum_helper(
                            self.settings,
                            gpg_operation=checksum_helper.VERIFY,
                            signature=signature,
                        )
                    elif f == gpkg_version_file:
                        # gpkg version file is not signed
                        checksum_info = checksum_helper(self.settings)
                    else:
                        raise MissingSignature(
                            f"{f} signature not found in {self.gpkg_file}"
                        )
                else:
                    checksum_info = checksum_helper(self.settings)

                # Verify current file checksum
                f_io = container.extractfile(f)
                while True:
                    buffer = f_io.read(HASHING_BLOCKSIZE)
                    if buffer:
                        checksum_info.update(buffer)
                    else:
                        checksum_info.finish()
                        break
                f_io.close()

                # At least one supported checksum must be checked
                verified_hash_count = 0
                for c in checksum_info.libs:
                    try:
                        if (
                            checksum_info.libs[c].hexdigest().lower()
                            == manifest_record[manifest_record.index(c) + 1].lower()
                        ):
                            verified_hash_count += 1
                        else:
                            raise DigestException(
                                f"{f} checksum mismatched in {self.gpkg_file}"
                            )
                    except KeyError:
                        # Checksum method not supported
                        pass

                if verified_hash_count < 1:
                    raise DigestException(
                        f"{f} no supported checksum found in {self.gpkg_file}"
                    )

                # Current file verified
                unverified_files.remove(f)
                unverified_manifest.remove(manifest_record)

        # Check if any files are IN the Manifest but NOT IN the binary package
        if len(unverified_manifest) != 0:
            raise DigestException(
                f"Missing files: {str(unverified_manifest)} in {self.gpkg_file}"
            )

        # Check if any files are NOT IN the Manifest but are IN the binary package
        if len(unverified_files) != 0:
            raise DigestException(
                f"Unknown files exists: {str(unverified_files)} in {self.gpkg_file}"
            )

        # Save current Manifest for other operations.
        self.manifest_old = manifest.copy()
        self.signature_exist = signature_exist
        self.prefix = prefix

    def _generate_metadata_from_dir(self, metadata_dir):
        """
        Read all files in metadata_dir and return as dict
        """
        metadata = {}
        metadata_dir = normalize_path(
            _unicode_decode(metadata_dir, encoding=_encodings["fs"], errors="strict")
        )
        for parent, dirs, files in os.walk(metadata_dir):
            for f in files:
                try:
                    f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
                except UnicodeDecodeError:
                    continue
                with open(os.path.join(parent, f), "rb") as metafile:
                    metadata[f] = metafile.read()
        return metadata

    def _get_binary_cmd(self, compression, mode):
        """
        Get command list from portage and try to match compressor
        """
        if compression not in _compressors:
            raise InvalidCompressionMethod(compression)

        compressor = _compressors[compression]
        if mode not in compressor:
            raise InvalidCompressionMethod(f"{compression}: {mode}")

        if mode == "compress" and (
            self.settings.get(f"BINPKG_COMPRESS_FLAGS_{compression.upper()}", None)
            is not None
        ):
            compressor["compress"] = compressor["compress"].replace(
                "${BINPKG_COMPRESS_FLAGS}",
                f"${{BINPKG_COMPRESS_FLAGS_{compression.upper()}}}",
            )

        cmd = compressor[mode].replace(
            "{JOBS}", str(makeopts_to_job_count(self.settings.get("MAKEOPTS", "1")))
        )
        cmd = shlex.split(varexpand(cmd, mydict=self.settings))

        # Filter empty elements that make Popen fail
        cmd = [x for x in cmd if x != ""]

        if (not cmd) and ((mode + "_alt") in compressor):
            cmd = shlex.split(
                varexpand(compressor[mode + "_alt"], mydict=self.settings)
            )
            cmd = [x for x in cmd if x != ""]

        if not cmd:
            raise CompressorNotFound(compression)
        if not find_binary(cmd[0]):
            raise CompressorNotFound(cmd[0])

        return cmd

    def _get_compression_cmd(self, compression=None):
        """
        Return compression command for Popen
        """
        if compression is None:
            compression = self.compression
        if compression is None:
            return None
        else:
            return self._get_binary_cmd(compression, "compress")

    def _get_decompression_cmd(self, compression=None):
        """
        Return decompression command for Popen
        """
        if compression is None:
            compression = self.compression
        if compression is None:
            return None
        else:
            return self._get_binary_cmd(compression, "decompress")

    def _get_tar_format(self, fileobj):
        """
        Try to detect tar version
        """
        old_position = fileobj.tell()
        fileobj.seek(0x101)
        magic = fileobj.read(8)
        fileobj.seek(0x9C)
        typeflag = fileobj.read(1)
        fileobj.seek(old_position)

        if magic == b"ustar  \x00":
            return tarfile.GNU_FORMAT
        elif magic == b"ustar\x0000":
            if typeflag == b"x" or typeflag == b"g":
                return tarfile.PAX_FORMAT
            else:
                return tarfile.USTAR_FORMAT

        return None

    def _get_tar_format_from_stats(
        self,
        image_max_prefix_length,
        image_max_name_length,
        image_max_linkname_length,
        image_max_file_size,
        image_total_size,
    ):
        """
        Choose the corresponding tar format according to
        the image information
        """
        # Max possible size in UStar is 8 GiB (8589934591 bytes)
        # stored in 11 octets
        # Use 8000000000, just in case we need add something extra

        # Total size > 8 GiB, container need use GNU tar format
        if image_total_size < 8000000000:
            container_tar_format = tarfile.USTAR_FORMAT
        else:
            container_tar_format = tarfile.GNU_FORMAT

        # Image at least one file > 8 GiB, image need use GNU tar format
        if image_max_file_size < 8000000000:
            image_tar_format = tarfile.USTAR_FORMAT
        else:
            image_tar_format = tarfile.GNU_FORMAT

        # UStar support max 155 prefix length, 100 file name and 100 link name,
        # ends with \x00. If any exceeded, failback to GNU format.
        if image_max_prefix_length >= 155:
            image_tar_format = tarfile.GNU_FORMAT

        if image_max_name_length >= 100:
            image_tar_format = tarfile.GNU_FORMAT

        if image_max_linkname_length >= 100:
            image_tar_format = tarfile.GNU_FORMAT
        return container_tar_format, image_tar_format

    def _check_pre_image_files(self, root_dir, image_prefix="image"):
        """
        Check the pre image files size and path, return the longest
        path length, largest single file size, and total files size.
        """
        image_prefix_length = len(image_prefix) + 1
        root_dir = os.path.join(
            normalize_path(
                _unicode_decode(root_dir, encoding=_encodings["fs"], errors="strict")
            ),
            "",
        )
        root_dir_length = len(
            _unicode_encode(root_dir, encoding=_encodings["fs"], errors="strict")
        )

        image_max_prefix_length = 0
        image_max_name_length = 0
        image_max_link_length = 0
        image_max_file_size = 0
        image_total_size = 0

        for parent, dirs, files in os.walk(root_dir):
            if portage.utf8_mode:
                parent = os.fsencode(parent)
                dirs = [os.fsencode(value) for value in dirs]
                files = [os.fsencode(value) for value in files]

            parent = _unicode_decode(parent, encoding=_encodings["fs"], errors="strict")
            for d in dirs:
                try:
                    d = _unicode_decode(d, encoding=_encodings["fs"], errors="strict")
                except UnicodeDecodeError as err:
                    writemsg(colorize("BAD", f"\n*** {err}\n\n"), noiselevel=-1)
                    raise

                d = os.path.join(parent, d)
                prefix_length = (
                    len(_unicode_encode(d, encoding=_encodings["fs"], errors="strict"))
                    - root_dir_length
                    + image_prefix_length
                )

                if os.path.islink(d):
                    path_link = os.readlink(d)
                    path_link_length = len(
                        _unicode_encode(
                            path_link, encoding=_encodings["fs"], errors="strict"
                        )
                    )
                    image_max_link_length = max(image_max_link_length, path_link_length)

                image_max_prefix_length = max(image_max_prefix_length, prefix_length)

            for f in files:
                try:
                    f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
                except UnicodeDecodeError as err:
                    writemsg(colorize("BAD", f"\n*** {err}\n\n"), noiselevel=-1)
                    raise

                filename_length = len(
                    _unicode_encode(f, encoding=_encodings["fs"], errors="strict")
                )
                image_max_name_length = max(image_max_name_length, filename_length)

                f = os.path.join(parent, f)
                path_length = (
                    len(_unicode_encode(f, encoding=_encodings["fs"], errors="strict"))
                    - root_dir_length
                    + image_prefix_length
                )

                file_stat = os.lstat(f)

                if stat.S_ISLNK(file_stat.st_mode):
                    path_link = os.readlink(f)
                    path_link_length = len(
                        os.fsencode(path_link)
                        if portage.utf8_mode
                        else _unicode_encode(
                            path_link, encoding=_encodings["fs"], errors="strict"
                        )
                    )
                elif file_stat.st_nlink > 1:
                    # Hardlink exists
                    path_link_length = path_length
                else:
                    path_link_length = 0

                image_max_link_length = max(image_max_link_length, path_link_length)

                if stat.S_ISLNK(file_stat.st_mode):
                    continue

                file_size = file_stat.st_size
                image_total_size += file_size
                image_max_file_size = max(image_max_file_size, file_size)

        return (
            image_max_prefix_length,
            image_max_name_length,
            image_max_link_length,
            image_max_file_size,
            image_total_size,
        )

    def _check_pre_quickpkg_files(
        self, contents, root, image_prefix="image", ignore_missing=False
    ):
        """
        Check the pre-quickpkg files size and path, return the longest
        path length, largest single file size, and total files size.
        """
        image_prefix_length = len(image_prefix) + 1
        root_dir = os.path.join(
            normalize_path(
                _unicode_decode(root, encoding=_encodings["fs"], errors="strict")
            ),
            "",
        )
        root_dir_length = len(
            _unicode_encode(root_dir, encoding=_encodings["fs"], errors="strict")
        )

        image_max_prefix_length = 0
        image_max_name_length = 0
        image_max_link_length = 0
        image_max_file_size = 0
        image_total_size = 0

        paths = list(contents)
        for path in paths:
            try:
                path = _unicode_decode(path, encoding=_encodings["fs"], errors="strict")
            except UnicodeDecodeError as err:
                writemsg(colorize("BAD", f"\n*** {err}\n\n"), noiselevel=-1)
                raise

            d, f = os.path.split(path)

            prefix_length = (
                len(_unicode_encode(d, encoding=_encodings["fs"], errors="strict"))
                - root_dir_length
                + image_prefix_length
            )
            image_max_prefix_length = max(image_max_prefix_length, prefix_length)

            filename_length = len(
                _unicode_encode(f, encoding=_encodings["fs"], errors="strict")
            )
            image_max_name_length = max(image_max_name_length, filename_length)

            path_length = (
                len(_unicode_encode(path, encoding=_encodings["fs"], errors="strict"))
                - root_dir_length
                + image_prefix_length
            )

            if not os.path.exists(path):
                if ignore_missing:
                    continue
                else:
                    raise FileNotFound(path)

            file_stat = os.lstat(path)

            if stat.S_ISLNK(file_stat.st_mode):
                path_link = os.readlink(path)
                path_link_length = len(
                    _unicode_encode(
                        path_link, encoding=_encodings["fs"], errors="strict"
                    )
                )
            elif file_stat.st_nlink > 1:
                # Hardlink exists
                path_link_length = path_length
            else:
                path_link_length = 0

            image_max_link_length = max(image_max_link_length, path_link_length)

            if os.path.isfile(path):
                if stat.S_ISLNK(file_stat.st_mode):
                    continue

                file_size = file_stat.st_size
                image_total_size += file_size
                if file_size > image_max_file_size:
                    image_max_file_size = file_size

        return (
            image_max_prefix_length,
            image_max_name_length,
            image_max_link_length,
            image_max_file_size,
            image_total_size,
        )

    def _create_tarinfo(self, file_name):
        """
        Create new tarinfo for the new file
        """
        if self.compression is None:
            ext = ""
        elif self.compression in self.ext_list:
            ext = self.ext_list[self.compression]
        else:
            raise InvalidCompressionMethod(self.compression)

        if self.basename:
            basename = self.basename
        elif self.prefix:
            basename = self.prefix
        else:
            raise InvalidBinaryPackageFormat("No basename or prefix specified")
        data_tarinfo = tarfile.TarInfo(os.path.join(basename, file_name + ".tar" + ext))
        return data_tarinfo

    def _extract_filename_compression(self, file_name):
        """
        Extract the file basename and compression method
        """
        file_name = os.path.basename(file_name)
        if file_name.endswith(".tar"):
            return file_name[:-4], None

        for compression in self.ext_list:
            if file_name.endswith(".tar" + self.ext_list[compression]):
                return (
                    file_name[: -len(".tar" + self.ext_list[compression])],
                    compression,
                )

        raise InvalidCompressionMethod(file_name)

    def _get_inner_tarinfo(self, tar, file_name):
        """
        Get inner tarinfo from given container.
        Will try get file_name from correct basename first,
        if it fails, try any file that have same name as file_name, and
        return the first one.
        """
        if self.gpkg_version not in (os.path.basename(f) for f in tar.getnames()):
            raise InvalidBinaryPackageFormat("Invalid gpkg file")

        if self.basename and self.prefix and not self.prefix.startswith(self.basename):
            writemsg(
                colorize("WARN", f"Package basename mismatched, using {self.prefix}\n")
            )

        all_files = tar.getmembers()
        for f in all_files:
            try:
                f_name, f_comp = self._extract_filename_compression(f.name)
            except InvalidCompressionMethod:
                continue

            if f_name == file_name:
                return f, f_comp

        # Not found
        raise FileNotFound(f"File Not found: {file_name}")
