parsers

soong_parser

logger: Logger

Logger.

Manifest

A Manifest (for AOSP) is an XML file listing all the projects used for a version of AOSP.

The repository listing all the manifests is found at : https://android.googlesource.com/platform/manifest/

This classes is a light wrapper around the XML File to query the parts of the manifest we are interested into.

Parameters:

Name	Type	Description	Default
manifest_content		The content of a manifest	required

__init__(self, manifest_content) special

Constructor method

Source code in bgraph/parsers/soong_parser.py

def __init__(self, manifest_content: str) -> None:
    """Constructor method"""
    try:
        self.xml: untangle.Element = untangle.parse(manifest_content)
    except xml.sax.SAXParseException as e:
        logger.exception(e)
        raise bgraph.exc.BGraphManifestException("Unable to load the manifest")

    if (
        getattr(self.xml, "manifest", False) is False
        or getattr(self.xml.manifest, "project", False) is False
    ):
        raise bgraph.exc.BGraphManifestException("Manifest misformed")

from_file(file_path) classmethod

Load a Manifest from a file

Parameters:

Name	Type	Description	Default
file_path	Union[str, pathlib.Path]	A PathLike object pointing to a file.	required

Returns:

Type	Description
Manifest	A Manifest class.

Source code in bgraph/parsers/soong_parser.py

@classmethod
def from_file(cls, file_path: Union[str, pathlib.Path]) -> "Manifest":
    """Load a Manifest from a file

    :param file_path: A PathLike object pointing to a file.
    :return: A `Manifest` class.
    """
    file_path = pathlib.Path(file_path)

    if file_path.is_file():
        return cls(manifest_content=file_path.as_posix())

    raise bgraph.exc.BGraphManifestException()

from_url(manifest_name, other_url=None) classmethod

Loads a manifest from an URL

Warning: This methods parses arbitrary data from an URL. Use with caution <!>

If the other_url parameter is set, this will be used as an absolute URL. Otherwise, it will fetch data from the googlesource servers.

Parameters:

Name	Type	Description	Default
manifest_name	str	Name of the manifest	required
other_url	Optional[str]	Use this url and not the one from Google	None

Returns:

Type	Description
Manifest	A Manifest class

Source code in bgraph/parsers/soong_parser.py

@classmethod
def from_url(
    cls, manifest_name: str, other_url: Optional[str] = None
) -> "Manifest":
    """Loads a manifest from an URL

    Warning: This methods parses arbitrary data from an URL. Use with caution <!>

    If the other_url parameter is set, this will be used as an absolute URL.
    Otherwise, it will fetch data from the googlesource servers.

    :param manifest_name: Name of the manifest
    :param other_url: Use this url and not the one from Google
    :return: A `Manifest` class
    """
    try:
        import requests
    except ImportError:
        raise bgraph.exc.BGraphManifestException(
            "Must have requests installed for this action"
        )

    if other_url is None:
        url_content = f"https://android.googlesource.com/platform/manifest/+/refs/heads/{manifest_name}/default.xml?format=TEXT"
    else:
        url_content = other_url

    try:
        response = requests.get(url_content)
    except requests.exceptions.RequestException as e:
        raise bgraph.exc.BGraphManifestException(e)

    try:
        xml_string: str = base64.decodebytes(response.content).decode(
            response.encoding
        )
    except TypeError as e:
        raise bgraph.exc.BGraphManifestException(e)

    return cls(manifest_content=xml_string)

get_projects(self)

Returns the list of the projects for a manifest.

Returns:

Type	Description
Dict[str, str]	A mapping between project name and project paths

Source code in bgraph/parsers/soong_parser.py

def get_projects(self) -> Dict[str, str]:
    """Returns the list of the projects for a manifest.

    :return: A mapping between project name and project paths
    """
    project_map: Dict[str, str] = {}
    for project in self.xml.manifest.project:
        project_name = project["name"]
        project_path = project["path"]

        if project_name is not None and project_path is not None:
            project_map[project_name] = project_path
        else:
            logger.warning(
                "Projet %s (@path: %s) is None", project_name, project_path
            )

    return project_map

SoongFileParser

Parser for soong files

Set and parse a soong file. This is a best effort parser and some edge cases are not supported.

Parameters:

Name	Type	Description	Default
(Dict)	variables	Mapping of variables and their value inside the Blueprint file	required

__init__(self, file_path=None, variables=None) special

Constructor

Source code in bgraph/parsers/soong_parser.py

def __init__(
    self,
    file_path: Optional[Union[pathlib.Path, str]] = None,
    variables: Dict[str, Any] = None,
):
    """Constructor"""
    self.variables: Dict[str, Any] = {}
    if variables is not None:
        self.variables = variables

    self.identifiers: Dict[str, Any] = {}

    self.sections: Dict[str, List[Section]] = collections.defaultdict(list)

    self.parser: pyparsing.ParserElement = self._init_parser()

    if file_path:
        self.parser.parseFile(pathlib.Path(file_path).as_posix())
        if self.identifiers or not (self.sections or self.variables):

            raise bgraph.exc.BGraphParserException("An error ocured during parsing")

parse_boolean(tokens) staticmethod

Helper method to parse boolean

Parameters:

Name	Type	Description	Default
tokens	List[Any]	List of tokens	required

Returns:

Type	Description
bool	A boolean

Exceptions:

Type	Description
BGraphParserException	When the boolean is not true or false

Source code in bgraph/parsers/soong_parser.py

@staticmethod
def parse_boolean(tokens: List[Any]) -> bool:
    """Helper method to parse boolean

    :param tokens: List of tokens
    :raises BGraphParserException: When the boolean is not true or false
    :return: A boolean
    """
    token = tokens[0]
    if token == "true":
        return True
    elif token == "false":
        return False

    raise bgraph.exc.BGraphParserException("Boolean exception")

parse_dict_def(tokens) staticmethod

Helper method to parse the dict definition

Source code in bgraph/parsers/soong_parser.py

@staticmethod
def parse_dict_def(tokens) -> Dict:
    """Helper method to parse the dict definition"""
    result_dict = {}
    for token in tokens:
        result_dict.update(token)

    return result_dict

parse_dict_field(tokens) staticmethod

Helper method to parse a map

Source code in bgraph/parsers/soong_parser.py

@staticmethod
def parse_dict_field(tokens) -> Dict:
    """Helper method to parse a map"""
    key = tokens[0]
    val = tokens[1] if len(tokens) == 2 else tokens[1:]

    return {key: val}

parse_integer(tokens) staticmethod

Helper method to parse integers

Parameters:

Name	Type	Description	Default
tokens	List[str]	Tokens	required

Returns:

Type	Description
int	An integer

Source code in bgraph/parsers/soong_parser.py

@staticmethod
def parse_integer(tokens: List[str]) -> int:
    """Helper method to parse integers

    :param tokens: Tokens
    :return: An integer
    """
    return int(tokens[0])

parse_list_concat(tokens) staticmethod

Helper for list concatenation

Parameters:

Name	Type	Description	Default
tokens	List[str]	Tokens	required

Returns:

Type	Description
List[Any]	A list of tokens

Source code in bgraph/parsers/soong_parser.py

@staticmethod
def parse_list_concat(tokens: List[str]) -> List[Any]:
    """Helper for list concatenation

    :param tokens: Tokens
    :return: A list of tokens
    """

    final_list: List[Any] = []
    for token in tokens:
        if type(token) is list:
            final_list.extend(token)
        elif type(token) is str:
            final_list.append(token)
        else:
            # Do not raise an exception as it may mess with pyparsing
            return []

    return final_list

parse_section(self, tokens)

Parse a section

Parameters:

Name	Type	Description	Default
tokens	List[str]	Tokens	required

Exceptions:

Type	Description
BGraphParserException	If the parsing of the section fails

Source code in bgraph/parsers/soong_parser.py

def parse_section(self, tokens: List[str]) -> None:
    """Parse a section

    :param tokens: Tokens
    :raises BGraphParserException: If the parsing of the section fails
    """
    section_name = None
    section_dict: Section = {SoongParser.SECTION_TYPE: tokens[0]}
    for token in tokens[1:]:
        if token in self.identifiers:

            if token == "name":
                section_name = self.identifiers[token]
            else:
                # TODO(dm) This will be resolved when we have a way to type hint a
                #   dict with dynamic values
                section_dict[token] = self.identifiers[token]  # type: ignore

            del self.identifiers[token]
        else:
            raise bgraph.exc.BGraphParserException(
                "Missing key {} in section {}".format(token, section_name)
            )

    # Each section *must* have a name
    if section_name is None:
        # Except soong_namespace ...
        if section_dict[SoongParser.SECTION_TYPE] in ("soong_namespace",):
            logger.debug("Found a soong namespace but it is not supported yet.")
            return

        raise bgraph.exc.BGraphParserException("Section has no attribute name")

    self.sections[section_name].append(section_dict)

parse_section_field(self, tokens)

Parse a section field

Example: name: "libartbase_defaults"

Source code in bgraph/parsers/soong_parser.py

def parse_section_field(self, tokens: List[str]) -> str:
    """Parse a section field

    Example:
        name: "libartbase_defaults"

    :return The name of the field
    """

    if len(tokens) == 2:
        self.identifiers[tokens[0]] = tokens[1]
    elif len(tokens) > 2:
        self.identifiers[tokens[0]] = tokens[1:]
    elif len(tokens) == 1:
        # FIX: handle empty definitions like whole_static_libs: []
        self.identifiers[tokens[0]] = []

    return tokens[0]

parse_string_concat(tokens) staticmethod

Helper method to concat string together

Parameters:

Name	Type	Description	Default
tokens	List[str]	Tokens	required

Returns:

Type	Description
Optional[str]	Optionnaly a string string

Source code in bgraph/parsers/soong_parser.py

@staticmethod
def parse_string_concat(tokens: List[str]) -> Optional[str]:
    """Helper method to concat string together

    :param tokens: Tokens
    :return: Optionnaly a string string
    """
    final_token = ""
    for token in tokens:
        if type(token) is str:
            final_token += token
        else:
            # Do not raise an exception as it may mess up with pyparsing
            return None

    return final_token

parse_variable_def(self, _, __, tokens, append=False)

Helper method to parse variable definition

Parameters:

Name	Type	Description	Default
_		N/A	required
__		N/A	required
tokens	List[Any]	Tokens to parse	required
append	bool	Should we appended the value to the existing one or not	False

Exceptions:

Type	Description
BGraphParserException	When the parsing fails

Source code in bgraph/parsers/soong_parser.py

def parse_variable_def(
    self, _, __, tokens: List[Any], append: bool = False
) -> None:
    """Helper method to parse variable definition

    :param _: N/A
    :param __: N/A
    :param tokens: Tokens to parse
    :param append: Should we appended the value to the existing one or not
    :raises BGraphParserException: When the parsing fails
    """
    variable_name = tokens[0]
    new_value = tokens[1] if len(tokens) == 2 else tokens[1:]
    if append is False:
        old_value = self.variables.get(variable_name, None)
        if old_value is not None and old_value != new_value:
            logger.debug("Overwritting variable - in debug, check if legit.")
            # raise bgraph.exc.BGraphParserException("Conflicting variables names")

        self.variables[variable_name] = new_value

    else:
        actual_value = self.variables.get(variable_name)
        if actual_value is None:
            raise bgraph.exc.BGraphParserException(
                "Missing previous variable during append"
            )

        if type(new_value) != type(actual_value):
            new_value = [new_value] if type(new_value) is str else new_value
            actual_value = (
                [actual_value] if type(actual_value) is str else actual_value
            )

        self.variables[variable_name] = actual_value + new_value

parse_variable_ref(self, tokens)

Helper method to parse variable reference

Parameters:

Name	Type	Description	Default
tokens	List[str]	Tokens to parse	required

Returns:

Type	Description
Any	The variable value

Source code in bgraph/parsers/soong_parser.py

def parse_variable_ref(self, tokens: List[str]) -> Any:
    """Helper method to parse variable reference

    :param tokens: Tokens to parse
    :raises: BGraphParserException When the variables is used before being defined

    :return: The variable value

    """
    var_name: str = tokens[0]
    if var_name not in self.variables:
        raise bgraph.exc.BGraphParserException("Missing variable ref var_name")

    return self.variables[var_name]

SoongParser

Soong Parser

This is the wrapper around the parser for Soong file (e.g. Android.bp)

Every section will be augmented with special keys (always prefixed with soong_parser).

DEFAULT_FILENAME: Final

Default name for Soong file.

file_listing: Dict[pathlib.Path, List[str]] property writable

A map of every paths and files inside the project. This is used to resolve wildcards in filenames for Soong.

Returns:

Type	Description
Dict[pathlib.Path, List[str]]	A maping between path and list of files inside an AOSP tree.

NATIVE_TYPES: List[str]

Type of section considered as "natives".

SECTION_PROJECT: Final

Name of the project for the current section.

SECTION_PROJECT_PATH: Final

Absolute path of the project in AOSP root tree.

SECTION_TYPE: Final

Type of the section (e.g. cc_library).

SOONG_FILE: Final

Absolute path of the Soong file in AOSP root tree.

__init__(self) special

Init method.

Source code in bgraph/parsers/soong_parser.py

def __init__(self) -> None:
    """Init method."""
    self.sections: Dict[str, List[Section]] = collections.defaultdict(list)
    self.variables: Dict[str, Any] = {}

    self._files_listing: Dict[pathlib.Path, List[str]] = {}

get_section(self, section_name, section_type=None)

Get a section from the project.

This is the main method of the class. It will also resolve the section defaults if any are found. Note that the name must be exact.

Parameters:

Name	Type	Description	Default
section_name	str	Name of the section	required
section_type	Optional[str]	Optional. Type of the section. If the type is defined, a single section will be returned of matching type. Otherwise all sections having the same name will be returned.	None

Returns:

Type	Description
Union[List[bgraph.types.Section], bgraph.types.Section]	A (list of) sections having the name asked.

Source code in bgraph/parsers/soong_parser.py

def get_section(
    self, section_name: str, section_type: Optional[str] = None
) -> Union[List[Section], Section]:
    """Get a section from the project.

    This is the main method of the class. It will also resolve the section defaults if
    any are found. Note that the name *must* be exact.

    :param section_name: Name of the section
    :param section_type: Optional. Type of the section. If the type is defined, a
        single section will be returned of matching type. Otherwise all sections
        having the same name will be returned.
    :return: A (list of) sections having the name asked.
    """

    sections: List[Section] = self._retrieve_section(section_name)
    if section_type is None:
        return sections

    for section in sections:
        if section[self.SECTION_TYPE] == section_type:
            return section

    raise bgraph.exc.BGraphMissingSectionException()

get_targets(self)

Compute the list of targets.

A section is considered to be a target if the section_type is a binary type. This method is for now pretty simplist but could be improved.

TODO: - add target parameter to filter targets - Filter also according to the value for the arch - add multi arch support - host/target difference

Returns:

Type	Description
List[str]	A list of section having a "binary" target.

Source code in bgraph/parsers/soong_parser.py

def get_targets(self) -> List[str]:
    """Compute the list of targets.

    A section is considered to be a target if the section_type is a binary type.
    This method is for now pretty simplist but could be improved.

    TODO:
        - add target parameter to filter targets
        - Filter also according to the value for the arch
        - add multi arch support
        - host/target difference

    :return: A list of section having a "binary" target.
    """
    target_list: List[str] = []
    for section_name in self.list_section(with_defaults=False):
        section_map: List[Section] = self.get_section(section_name)

        for section in section_map:
            section_type = section.get(self.SECTION_TYPE)

            if section_type in [
                "cc_library",
                "cc_library_shared",
                "cc_library_static",
            ]:
                # The target is actually the name of the section. Manual says it can
                # be overriden but I did not find any evidence of that.
                # TODO(dm) : see if the name if overriden & check if the lib is not
                #  disabled for target ? (how?)
                target_list.append(section_name)
            elif section_type in [
                "cc_binary",
            ]:
                target_list.append(section_name)

    return target_list

list_section(self, with_defaults=False)

List sections found in AOSP.

Parameters:

Name	Type	Description	Default
with_defaults	bool	Also include defaults sections in the results	False

Returns:

Type	Description
List[str]	A list of sections

Source code in bgraph/parsers/soong_parser.py

def list_section(self, with_defaults: bool = False) -> List[str]:
    """List sections found in AOSP.

    :param with_defaults: Also include defaults sections in the results
    :return: A list of sections
    """
    section_list: List[str] = []
    for section_name, targets in self.sections.items():
        if with_defaults or any(
            "default" not in target.get(self.SECTION_TYPE, "") for target in targets
        ):
            section_list.append(section_name)

    return section_list

parse_aosp(self, aosp_directory, file_name=None, project_map=None)

Parses an AOSP tree.

This methods only needs the soong file to be present so a partial checkout is enough to create the listing.

The project map is needed because it needs to know the root tree of a project.

Parameters:

Name	Type	Description	Default
aosp_directory	Union[str, pathlib.Path]	Root tree of AOSP	required
file_name	Optional[str]	Optional Name of file	None
project_map	Dict	A map of project name / project path	None

Source code in bgraph/parsers/soong_parser.py

def parse_aosp(
    self,
    aosp_directory: Union[str, pathlib.Path],
    file_name: Optional[str] = None,
    project_map: Dict = None,
) -> None:
    """Parses an AOSP tree.

    This methods only needs the soong file to be present so a partial checkout is
    enough to create the listing.

    The project map is needed because it needs to know the root tree of a project.

    :param aosp_directory: Root tree of AOSP
    :param file_name: Optional Name of file
    :param project_map: A map of project name / project path
    """
    if file_name is None:
        file_name = self.DEFAULT_FILENAME

    if project_map is None:
        raise bgraph.exc.BGraphParserException("Missing project map.")

    aosp_directory = pathlib.Path(aosp_directory)
    for project_name, relative_path in project_map.items():
        project_path = aosp_directory / relative_path
        self.parse_project(
            project_directory=project_path,
            file_name=file_name,
            project_name=project_name,
        )

parse_file(self, file_path, project_name=None, project_path=None, project_variables=None)

Parse a file (e.g. an Android.bp) and update the current class.

Note: This will silently fails if the file is misformed.

Parameters:

Name	Type	Description	Default
file_path	Union[str, pathlib.Path]	Path towards the file	required
project_name	str	Optional. Name of the current project	None
project_path	Optional[pathlib.Path]	Optional. Path to the root of the project	None
project_variables	Dict[str, Any]	Variables already set for the project	None

Source code in bgraph/parsers/soong_parser.py

def parse_file(
    self,
    file_path: Union[str, pathlib.Path],
    project_name: str = None,
    project_path: Optional[pathlib.Path] = None,
    project_variables: Dict[str, Any] = None,
) -> None:
    """Parse a file (e.g. an Android.bp) and update the current class.

    Note: This will silently fails if the file is misformed.

    :param file_path: Path towards the file
    :param project_name: Optional. Name of the current project
    :param project_path: Optional. Path to the root of the project
    :param project_variables: Variables already set for the project
    """
    if project_variables is None:
        project_variables = {}

    try:
        parser = SoongFileParser(file_path, project_variables)
    except bgraph.exc.BGraphParserException:
        # The parser is a best effort one. If it fails, do not try to hard but
        # report the error if in DEBUG mode.
        logger.debug("Failed to parse %s", file_path)
        return

    # If we are doing a parsing of a project, we want to store more information on
    # the sections, namely where was the initial file located and the root source of
    # the project. This will be handy when resolving relative paths.
    if project_name is not None:
        for section_name, sections in parser.sections.items():
            for section in sections:
                section[self.SECTION_PROJECT] = project_name
                section[self.SOONG_FILE] = pathlib.Path(file_path)

                if project_path is not None:
                    section[self.SECTION_PROJECT_PATH] = project_path

                self.sections[section_name].append(section)

    project_variables.update(parser.variables)
    self.variables.update(parser.variables)

parse_project(self, project_directory, project_name, file_name=None)

Parse a project inside AOSP

This methods expects the project to be an AOSP project (e.g. an entry in the manifest list of projects).

Parameters:

Name	Type	Description	Default
project_directory	Union[str, pathlib.Path]	Path towards the project	required
project_name	str	Name of the project	required
file_name	Optional[str]	Name of the soong files	None

Source code in bgraph/parsers/soong_parser.py

def parse_project(
    self,
    project_directory: Union[str, pathlib.Path],
    project_name: str,
    file_name: Optional[str] = None,
) -> None:
    """Parse a project inside AOSP

    This methods expects the project to be an AOSP project (e.g. an entry in the
    manifest list of projects).

    :param project_directory: Path towards the project
    :param project_name: Name of the project
    :param file_name: Name of the soong files
    """
    if file_name is None:
        file_name = self.DEFAULT_FILENAME

    project_directory = pathlib.Path(project_directory)
    project_variables: Dict[str, Any] = {}
    for soong_file in project_directory.rglob(file_name):
        self.parse_file(
            file_path=soong_file,
            project_name=project_name,
            project_path=project_directory,
            project_variables=project_variables,
        )

    # Sometimes in Android, a project may be have additional "generic" components
    # We try to include also Build Files files from those directories here to handle this case
    # This is a *dirty* hack and it should not be necessary when the project is from
    # the manifest.
    for soong_file in (project_directory.parent / "generic").rglob(file_name):
        self.parse_file(file_path=soong_file)