diff --git a/odoo_repository/README.rst b/odoo_repository/README.rst new file mode 100644 index 0000000..0695239 --- /dev/null +++ b/odoo_repository/README.rst @@ -0,0 +1,90 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + +====================== +Odoo Repositories Data +====================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:f4c98a6da3a2dd638fb292cbbae95681079c337555ced17cd560dd565e89dd17 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fmodule--composition--analysis-lightgray.png?logo=github + :target: https://github.com/OCA/module-composition-analysis/tree/16.0/odoo_repository + :alt: OCA/module-composition-analysis +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/module-composition-analysis-16-0/module-composition-analysis-16-0-odoo_repository + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/module-composition-analysis&target_branch=16.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Base module to host data collected from Odoo repositories. + +It allows you to: + +- declare the Odoo versions (last 3 versions by default) +- declare repositories containing modules (Odoo and OCA repositories + included by default) +- scan these repositories to collect modules informations per Odoo + version + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* Camptocamp + +Contributors +------------ + +- Camptocamp + + - Sébastien Alix + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/module-composition-analysis `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/odoo_repository/__init__.py b/odoo_repository/__init__.py new file mode 100644 index 0000000..f7209b1 --- /dev/null +++ b/odoo_repository/__init__.py @@ -0,0 +1,2 @@ +from . import models +from . import controllers diff --git a/odoo_repository/__manifest__.py b/odoo_repository/__manifest__.py new file mode 100644 index 0000000..8cbc14c --- /dev/null +++ b/odoo_repository/__manifest__.py @@ -0,0 +1,58 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +{ + "name": "Odoo Repositories Data", + "summary": "Base module to host data collected from Odoo repositories.", + "version": "18.0.1.0.0", + "category": "Tools", + "author": "Camptocamp, Odoo Community Association (OCA)", + "website": "https://github.com/OCA/module-composition-analysis", + "data": [ + "security/res_groups.xml", + "security/ir.model.access.csv", + "data/ir_cron.xml", + "data/odoo_module.xml", + "data/odoo_repository_org.xml", + "data/odoo_repository_addons_path.xml", + "data/odoo_repository.xml", + "data/odoo.repository.csv", + "data/odoo_branch.xml", + "data/queue_job.xml", + "views/menu.xml", + "views/authentication_token.xml", + "views/ssh_key.xml", + "views/odoo_author.xml", + "views/odoo_branch.xml", + "views/odoo_license.xml", + "views/odoo_maintainer.xml", + "views/odoo_module.xml", + "views/odoo_module_branch.xml", + "views/odoo_module_category.xml", + "views/odoo_module_dev_status.xml", + "views/odoo_python_dependency.xml", + "views/odoo_repository.xml", + "views/odoo_repository_addons_path.xml", + "views/odoo_repository_branch.xml", + "views/odoo_repository_org.xml", + "views/res_config_settings.xml", + ], + "installable": True, + "application": True, + "depends": [ + # core + "base_sparse_field", + # OCA/server-tools + "base_time_window", + # OCA/queue + "queue_job", + ], + "external_dependencies": { + "python": [ + "gitpython", + "odoo-addons-parser", + # TODO to publish + # "odoo-repository-scanner" + ], + }, + "license": "AGPL-3", +} diff --git a/odoo_repository/controllers/__init__.py b/odoo_repository/controllers/__init__.py new file mode 100644 index 0000000..12a7e52 --- /dev/null +++ b/odoo_repository/controllers/__init__.py @@ -0,0 +1 @@ +from . import main diff --git a/odoo_repository/controllers/main.py b/odoo_repository/controllers/main.py new file mode 100644 index 0000000..78d2496 --- /dev/null +++ b/odoo_repository/controllers/main.py @@ -0,0 +1,32 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import json + +from odoo import http + + +class OdooRepository(http.Controller): + @http.route("/odoo-repository/data", type="http", auth="none", csrf=False) + def index(self, orgs: str = None, repositories: str = None, branches: str = None): + """Returns modules data as JSON. + + This endpoint is used by secondary nodes that want to sync the data + collected by the main node. + + Parameters are strings that can be set with multiple values separated + by commas, e.g. `branches="15.0,16.0"`. + """ + if orgs: + orgs = orgs.split(",") + if repositories: + repositories = repositories.split(",") + if branches: + branches = branches.split(",") + data = ( + http.request.env["odoo.module.branch"] + .sudo() + ._get_modules_data(orgs=orgs, repositories=repositories, branches=branches) + ) + headers = {"Content-Type": "application/json"} + return http.request.make_response(json.dumps(data), headers) diff --git a/odoo_repository/data/ir_cron.xml b/odoo_repository/data/ir_cron.xml new file mode 100644 index 0000000..9a55c90 --- /dev/null +++ b/odoo_repository/data/ir_cron.xml @@ -0,0 +1,32 @@ + + + + + Odoo MCA - Scanner + 1 + days + + + + code + model.cron_scanner() + + + + Odoo MCA - Fetch data from main node + 1 + days + + + + code + model.cron_fetch_data() + + diff --git a/odoo_repository/data/odoo.repository.csv b/odoo_repository/data/odoo.repository.csv new file mode 100644 index 0000000..652e0c9 --- /dev/null +++ b/odoo_repository/data/odoo.repository.csv @@ -0,0 +1,216 @@ +id,org_id:id,name,repo_url,clone_url,repo_type +repo_oca_account_analytic,odoo_repository_org_oca,account-analytic,https://github.com/OCA/account-analytic,https://github.com/OCA/account-analytic,github +repo_oca_account_budgeting,odoo_repository_org_oca,account-budgeting,https://github.com/OCA/account-budgeting,https://github.com/OCA/account-budgeting,github +repo_oca_account_closing,odoo_repository_org_oca,account-closing,https://github.com/OCA/account-closing,https://github.com/OCA/account-closing,github +repo_oca_account_consolidation,odoo_repository_org_oca,account-consolidation,https://github.com/OCA/account-consolidation,https://github.com/OCA/account-consolidation,github +repo_oca_account_financial_reporting,odoo_repository_org_oca,account-financial-reporting,https://github.com/OCA/account-financial-reporting,https://github.com/OCA/account-financial-reporting,github +repo_oca_account_financial_tools,odoo_repository_org_oca,account-financial-tools,https://github.com/OCA/account-financial-tools,https://github.com/OCA/account-financial-tools,github +repo_oca_account_fiscal_rule,odoo_repository_org_oca,account-fiscal-rule,https://github.com/OCA/account-fiscal-rule,https://github.com/OCA/account-fiscal-rule,github +repo_oca_account_invoice_reporting,odoo_repository_org_oca,account-invoice-reporting,https://github.com/OCA/account-invoice-reporting,https://github.com/OCA/account-invoice-reporting,github +repo_oca_account_invoicing,odoo_repository_org_oca,account-invoicing,https://github.com/OCA/account-invoicing,https://github.com/OCA/account-invoicing,github +repo_oca_account_payment,odoo_repository_org_oca,account-payment,https://github.com/OCA/account-payment,https://github.com/OCA/account-payment,github +repo_oca_account_reconcile,odoo_repository_org_oca,account-reconcile,https://github.com/OCA/account-reconcile,https://github.com/OCA/account-reconcile,github +repo_oca_agreement,odoo_repository_org_oca,agreement,https://github.com/OCA/agreement,https://github.com/OCA/agreement,github +repo_oca_apps_store,odoo_repository_org_oca,apps-store,https://github.com/OCA/apps-store,https://github.com/OCA/apps-store,github +repo_oca_bank_payment,odoo_repository_org_oca,bank-payment,https://github.com/OCA/bank-payment,https://github.com/OCA/bank-payment,github +repo_oca_bank_payment_alternative,odoo_repository_org_oca,bank-payment-alternative,https://github.com/OCA/bank-payment-alternative,https://github.com/OCA/bank-payment-alternative,github +repo_oca_bank_statement_import,odoo_repository_org_oca,bank-statement-import,https://github.com/OCA/bank-statement-import,https://github.com/OCA/bank-statement-import,github +repo_oca_brand,odoo_repository_org_oca,brand,https://github.com/OCA/brand,https://github.com/OCA/brand,github +repo_oca_business_requirement,odoo_repository_org_oca,business-requirement,https://github.com/OCA/business-requirement,https://github.com/OCA/business-requirement,github +repo_oca_calendar,odoo_repository_org_oca,calendar,https://github.com/OCA/calendar,https://github.com/OCA/calendar,github +repo_oca_commission,odoo_repository_org_oca,commission,https://github.com/OCA/commission,https://github.com/OCA/commission,github +repo_oca_community_data_files,odoo_repository_org_oca,community-data-files,https://github.com/OCA/community-data-files,https://github.com/OCA/community-data-files,github +repo_oca_connector,odoo_repository_org_oca,connector,https://github.com/OCA/connector,https://github.com/OCA/connector,github +repo_oca_connector_accountedge,odoo_repository_org_oca,connector-accountedge,https://github.com/OCA/connector-accountedge,https://github.com/OCA/connector-accountedge,github +repo_oca_connector_cmis,odoo_repository_org_oca,connector-cmis,https://github.com/OCA/connector-cmis,https://github.com/OCA/connector-cmis,github +repo_oca_connector_ecommerce,odoo_repository_org_oca,connector-ecommerce,https://github.com/OCA/connector-ecommerce,https://github.com/OCA/connector-ecommerce,github +repo_oca_connector_infor,odoo_repository_org_oca,connector-infor,https://github.com/OCA/connector-infor,https://github.com/OCA/connector-infor,github +repo_oca_connector_interfaces,odoo_repository_org_oca,connector-interfaces,https://github.com/OCA/connector-interfaces,https://github.com/OCA/connector-interfaces,github +repo_oca_connector_jira,odoo_repository_org_oca,connector-jira,https://github.com/OCA/connector-jira,https://github.com/OCA/connector-jira,github +repo_oca_connector_lengow,odoo_repository_org_oca,connector-lengow,https://github.com/OCA/connector-lengow,https://github.com/OCA/connector-lengow,github +repo_oca_connector_lims,odoo_repository_org_oca,connector-lims,https://github.com/OCA/connector-lims,https://github.com/OCA/connector-lims,github +repo_oca_connector_magento,odoo_repository_org_oca,connector-magento,https://github.com/OCA/connector-magento,https://github.com/OCA/connector-magento,github +repo_oca_connector_magento_php_extension,odoo_repository_org_oca,connector-magento-php-extension,https://github.com/OCA/connector-magento-php-extension,https://github.com/OCA/connector-magento-php-extension,github +repo_oca_connector_odoo2odoo,odoo_repository_org_oca,connector-odoo2odoo,https://github.com/OCA/connector-odoo2odoo,https://github.com/OCA/connector-odoo2odoo,github +repo_oca_connector_prestashop,odoo_repository_org_oca,connector-prestashop,https://github.com/OCA/connector-prestashop,https://github.com/OCA/connector-prestashop,github +repo_oca_connector_redmine,odoo_repository_org_oca,connector-redmine,https://github.com/OCA/connector-redmine,https://github.com/OCA/connector-redmine,github +repo_oca_connector_sage,odoo_repository_org_oca,connector-sage,https://github.com/OCA/connector-sage,https://github.com/OCA/connector-sage,github +repo_oca_connector_salesforce,odoo_repository_org_oca,connector-salesforce,https://github.com/OCA/connector-salesforce,https://github.com/OCA/connector-salesforce,github +repo_oca_connector_spscommerce,odoo_repository_org_oca,connector-spscommerce,https://github.com/OCA/connector-spscommerce,https://github.com/OCA/connector-spscommerce,github +repo_oca_connector_telephony,odoo_repository_org_oca,connector-telephony,https://github.com/OCA/connector-telephony,https://github.com/OCA/connector-telephony,github +repo_oca_connector_woocommerce,odoo_repository_org_oca,connector-woocommerce,https://github.com/OCA/connector-woocommerce,https://github.com/OCA/connector-woocommerce,github +repo_oca_contract,odoo_repository_org_oca,contract,https://github.com/OCA/contract,https://github.com/OCA/contract,github +repo_oca_cooperative,odoo_repository_org_oca,cooperative,https://github.com/OCA/cooperative,https://github.com/OCA/cooperative,github +repo_oca_credit_control,odoo_repository_org_oca,credit-control,https://github.com/OCA/credit-control,https://github.com/OCA/credit-control,github +repo_oca_crm,odoo_repository_org_oca,crm,https://github.com/OCA/crm,https://github.com/OCA/crm,github +repo_oca_currency,odoo_repository_org_oca,currency,https://github.com/OCA/currency,https://github.com/OCA/currency,github +repo_oca_data_protection,odoo_repository_org_oca,data-protection,https://github.com/OCA/data-protection,https://github.com/OCA/data-protection,github +repo_oca_ddmrp,odoo_repository_org_oca,ddmrp,https://github.com/OCA/ddmrp,https://github.com/OCA/ddmrp,github +repo_oca_delivery_carrier,odoo_repository_org_oca,delivery-carrier,https://github.com/OCA/delivery-carrier,https://github.com/OCA/delivery-carrier,github +repo_oca_department,odoo_repository_org_oca,department,https://github.com/OCA/department,https://github.com/OCA/department,github +repo_oca_dms,odoo_repository_org_oca,dms,https://github.com/OCA/dms,https://github.com/OCA/dms,github +repo_oca_donation,odoo_repository_org_oca,donation,https://github.com/OCA/donation,https://github.com/OCA/donation,github +repo_oca_e_commerce,odoo_repository_org_oca,e-commerce,https://github.com/OCA/e-commerce,https://github.com/OCA/e-commerce,github +repo_oca_e_learning,odoo_repository_org_oca,e-learning,https://github.com/OCA/e-learning,https://github.com/OCA/e-learning,github +repo_oca_edi,odoo_repository_org_oca,edi,https://github.com/OCA/edi,https://github.com/OCA/edi,github +repo_oca_edi_framework,odoo_repository_org_oca,edi-framework,https://github.com/OCA/edi-framework,https://github.com/OCA/edi-framework,github +repo_oca_event,odoo_repository_org_oca,event,https://github.com/OCA/event,https://github.com/OCA/event,github +repo_oca_field_service,odoo_repository_org_oca,field-service,https://github.com/OCA/field-service,https://github.com/OCA/field-service,github +repo_oca_fleet,odoo_repository_org_oca,fleet,https://github.com/OCA/fleet,https://github.com/OCA/fleet,github +repo_oca_geospatial,odoo_repository_org_oca,geospatial,https://github.com/OCA/geospatial,https://github.com/OCA/geospatial,github +repo_oca_helpdesk,odoo_repository_org_oca,helpdesk,https://github.com/OCA/helpdesk,https://github.com/OCA/helpdesk,github +repo_oca_hr,odoo_repository_org_oca,hr,https://github.com/OCA/hr,https://github.com/OCA/hr,github +repo_oca_hr_attendance,odoo_repository_org_oca,hr-attendance,https://github.com/OCA/hr-attendance,https://github.com/OCA/hr-attendance,github +repo_oca_hr_expense,odoo_repository_org_oca,hr-expense,https://github.com/OCA/hr-expense,https://github.com/OCA/hr-expense,github +repo_oca_hr_holidays,odoo_repository_org_oca,hr-holidays,https://github.com/OCA/hr-holidays,https://github.com/OCA/hr-holidays,github +repo_oca_infrastructure,odoo_repository_org_oca,infrastructure,https://github.com/OCA/infrastructure,https://github.com/OCA/infrastructure,github +repo_oca_interface_github,odoo_repository_org_oca,interface-github,https://github.com/OCA/interface-github,https://github.com/OCA/interface-github,github +repo_oca_intrastat_extrastat,odoo_repository_org_oca,intrastat-extrastat,https://github.com/OCA/intrastat-extrastat,https://github.com/OCA/intrastat-extrastat,github +repo_oca_iot,odoo_repository_org_oca,iot,https://github.com/OCA/iot,https://github.com/OCA/iot,github +repo_oca_knowledge,odoo_repository_org_oca,knowledge,https://github.com/OCA/knowledge,https://github.com/OCA/knowledge,github +repo_oca_l10n_argentina,odoo_repository_org_oca,l10n-argentina,https://github.com/OCA/l10n-argentina,https://github.com/OCA/l10n-argentina,github +repo_oca_l10n_austria,odoo_repository_org_oca,l10n-austria,https://github.com/OCA/l10n-austria,https://github.com/OCA/l10n-austria,github +repo_oca_l10n_belarus,odoo_repository_org_oca,l10n-belarus,https://github.com/OCA/l10n-belarus,https://github.com/OCA/l10n-belarus,github +repo_oca_l10n_belgium,odoo_repository_org_oca,l10n-belgium,https://github.com/OCA/l10n-belgium,https://github.com/OCA/l10n-belgium,github +repo_oca_l10n_brazil,odoo_repository_org_oca,l10n-brazil,https://github.com/OCA/l10n-brazil,https://github.com/OCA/l10n-brazil,github +repo_oca_l10n_cambodia,odoo_repository_org_oca,l10n-cambodia,https://github.com/OCA/l10n-cambodia,https://github.com/OCA/l10n-cambodia,github +repo_oca_l10n_canada,odoo_repository_org_oca,l10n-canada,https://github.com/OCA/l10n-canada,https://github.com/OCA/l10n-canada,github +repo_oca_l10n_chile,odoo_repository_org_oca,l10n-chile,https://github.com/OCA/l10n-chile,https://github.com/OCA/l10n-chile,github +repo_oca_l10n_china,odoo_repository_org_oca,l10n-china,https://github.com/OCA/l10n-china,https://github.com/OCA/l10n-china,github +repo_oca_l10n_colombia,odoo_repository_org_oca,l10n-colombia,https://github.com/OCA/l10n-colombia,https://github.com/OCA/l10n-colombia,github +repo_oca_l10n_costa_rica,odoo_repository_org_oca,l10n-costa-rica,https://github.com/OCA/l10n-costa-rica,https://github.com/OCA/l10n-costa-rica,github +repo_oca_l10n_croatia,odoo_repository_org_oca,l10n-croatia,https://github.com/OCA/l10n-croatia,https://github.com/OCA/l10n-croatia,github +repo_oca_l10n_ecuador,odoo_repository_org_oca,l10n-ecuador,https://github.com/OCA/l10n-ecuador,https://github.com/OCA/l10n-ecuador,github +repo_oca_l10n_estonia,odoo_repository_org_oca,l10n-estonia,https://github.com/OCA/l10n-estonia,https://github.com/OCA/l10n-estonia,github +repo_oca_l10n_ethiopia,odoo_repository_org_oca,l10n-ethiopia,https://github.com/OCA/l10n-ethiopia,https://github.com/OCA/l10n-ethiopia,github +repo_oca_l10n_finland,odoo_repository_org_oca,l10n-finland,https://github.com/OCA/l10n-finland,https://github.com/OCA/l10n-finland,github +repo_oca_l10n_france,odoo_repository_org_oca,l10n-france,https://github.com/OCA/l10n-france,https://github.com/OCA/l10n-france,github +repo_oca_l10n_germany,odoo_repository_org_oca,l10n-germany,https://github.com/OCA/l10n-germany,https://github.com/OCA/l10n-germany,github +repo_oca_l10n_greece,odoo_repository_org_oca,l10n-greece,https://github.com/OCA/l10n-greece,https://github.com/OCA/l10n-greece,github +repo_oca_l10n_india,odoo_repository_org_oca,l10n-india,https://github.com/OCA/l10n-india,https://github.com/OCA/l10n-india,github +repo_oca_l10n_indonesia,odoo_repository_org_oca,l10n-indonesia,https://github.com/OCA/l10n-indonesia,https://github.com/OCA/l10n-indonesia,github +repo_oca_l10n_iran,odoo_repository_org_oca,l10n-iran,https://github.com/OCA/l10n-iran,https://github.com/OCA/l10n-iran,github +repo_oca_l10n_ireland,odoo_repository_org_oca,l10n-ireland,https://github.com/OCA/l10n-ireland,https://github.com/OCA/l10n-ireland,github +repo_oca_l10n_italy,odoo_repository_org_oca,l10n-italy,https://github.com/OCA/l10n-italy,https://github.com/OCA/l10n-italy,github +repo_oca_l10n_japan,odoo_repository_org_oca,l10n-japan,https://github.com/OCA/l10n-japan,https://github.com/OCA/l10n-japan,github +repo_oca_l10n_luxemburg,odoo_repository_org_oca,l10n-luxemburg,https://github.com/OCA/l10n-luxemburg,https://github.com/OCA/l10n-luxemburg,github +repo_oca_l10n_macedonia,odoo_repository_org_oca,l10n-macedonia,https://github.com/OCA/l10n-macedonia,https://github.com/OCA/l10n-macedonia,github +repo_oca_l10n_mexico,odoo_repository_org_oca,l10n-mexico,https://github.com/OCA/l10n-mexico,https://github.com/OCA/l10n-mexico,github +repo_oca_l10n_morocco,odoo_repository_org_oca,l10n-morocco,https://github.com/OCA/l10n-morocco,https://github.com/OCA/l10n-morocco,github +repo_oca_l10n_netherlands,odoo_repository_org_oca,l10n-netherlands,https://github.com/OCA/l10n-netherlands,https://github.com/OCA/l10n-netherlands,github +repo_oca_l10n_norway,odoo_repository_org_oca,l10n-norway,https://github.com/OCA/l10n-norway,https://github.com/OCA/l10n-norway,github +repo_oca_l10n_peru,odoo_repository_org_oca,l10n-peru,https://github.com/OCA/l10n-peru,https://github.com/OCA/l10n-peru,github +repo_oca_l10n_poland,odoo_repository_org_oca,l10n-poland,https://github.com/OCA/l10n-poland,https://github.com/OCA/l10n-poland,github +repo_oca_l10n_portugal,odoo_repository_org_oca,l10n-portugal,https://github.com/OCA/l10n-portugal,https://github.com/OCA/l10n-portugal,github +repo_oca_l10n_romania,odoo_repository_org_oca,l10n-romania,https://github.com/OCA/l10n-romania,https://github.com/OCA/l10n-romania,github +repo_oca_l10n_russia,odoo_repository_org_oca,l10n-russia,https://github.com/OCA/l10n-russia,https://github.com/OCA/l10n-russia,github +repo_oca_l10n_slovenia,odoo_repository_org_oca,l10n-slovenia,https://github.com/OCA/l10n-slovenia,https://github.com/OCA/l10n-slovenia,github +repo_oca_l10n_spain,odoo_repository_org_oca,l10n-spain,https://github.com/OCA/l10n-spain,https://github.com/OCA/l10n-spain,github +repo_oca_l10n_switzerland,odoo_repository_org_oca,l10n-switzerland,https://github.com/OCA/l10n-switzerland,https://github.com/OCA/l10n-switzerland,github +repo_oca_l10n_taiwan,odoo_repository_org_oca,l10n-taiwan,https://github.com/OCA/l10n-taiwan,https://github.com/OCA/l10n-taiwan,github +repo_oca_l10n_thailand,odoo_repository_org_oca,l10n-thailand,https://github.com/OCA/l10n-thailand,https://github.com/OCA/l10n-thailand,github +repo_oca_l10n_turkey,odoo_repository_org_oca,l10n-turkey,https://github.com/OCA/l10n-turkey,https://github.com/OCA/l10n-turkey,github +repo_oca_l10n_ukraine,odoo_repository_org_oca,l10n-ukraine,https://github.com/OCA/l10n-ukraine,https://github.com/OCA/l10n-ukraine,github +repo_oca_l10n_united_kingdom,odoo_repository_org_oca,l10n-united-kingdom,https://github.com/OCA/l10n-united-kingdom,https://github.com/OCA/l10n-united-kingdom,github +repo_oca_l10n_uruguay,odoo_repository_org_oca,l10n-uruguay,https://github.com/OCA/l10n-uruguay,https://github.com/OCA/l10n-uruguay,github +repo_oca_l10n_usa,odoo_repository_org_oca,l10n-usa,https://github.com/OCA/l10n-usa,https://github.com/OCA/l10n-usa,github +repo_oca_l10n_venezuela,odoo_repository_org_oca,l10n-venezuela,https://github.com/OCA/l10n-venezuela,https://github.com/OCA/l10n-venezuela,github +repo_oca_l10n_vietnam,odoo_repository_org_oca,l10n-vietnam,https://github.com/OCA/l10n-vietnam,https://github.com/OCA/l10n-vietnam,github +repo_oca_mail,odoo_repository_org_oca,mail,https://github.com/OCA/mail,https://github.com/OCA/mail,github +repo_oca_maintenance,odoo_repository_org_oca,maintenance,https://github.com/OCA/maintenance,https://github.com/OCA/maintenance,github +repo_oca_management_system,odoo_repository_org_oca,management-system,https://github.com/OCA/management-system,https://github.com/OCA/management-system,github +repo_oca_manufacture,odoo_repository_org_oca,manufacture,https://github.com/OCA/manufacture,https://github.com/OCA/manufacture,github +repo_oca_manufacture_reporting,odoo_repository_org_oca,manufacture-reporting,https://github.com/OCA/manufacture-reporting,https://github.com/OCA/manufacture-reporting,github +repo_oca_margin_analysis,odoo_repository_org_oca,margin-analysis,https://github.com/OCA/margin-analysis,https://github.com/OCA/margin-analysis,github +repo_oca_mis_builder,odoo_repository_org_oca,mis-builder,https://github.com/OCA/mis-builder,https://github.com/OCA/mis-builder,github +repo_oca_mis_builder_contrib,odoo_repository_org_oca,mis-builder-contrib,https://github.com/OCA/mis-builder-contrib,https://github.com/OCA/mis-builder-contrib,github +repo_oca_multi_company,odoo_repository_org_oca,multi-company,https://github.com/OCA/multi-company,https://github.com/OCA/multi-company,github +repo_oca_oca_custom,odoo_repository_org_oca,oca-custom,https://github.com/OCA/oca-custom,https://github.com/OCA/oca-custom,github +repo_oca_odoo_pim,odoo_repository_org_oca,odoo-pim,https://github.com/OCA/odoo-pim,https://github.com/OCA/odoo-pim,github +repo_oca_operating_unit,odoo_repository_org_oca,operating-unit,https://github.com/OCA/operating-unit,https://github.com/OCA/operating-unit,github +repo_oca_partner_contact,odoo_repository_org_oca,partner-contact,https://github.com/OCA/partner-contact,https://github.com/OCA/partner-contact,github +repo_oca_payroll,odoo_repository_org_oca,payroll,https://github.com/OCA/payroll,https://github.com/OCA/payroll,github +repo_oca_pms,odoo_repository_org_oca,pms,https://github.com/OCA/pms,https://github.com/OCA/pms,github +repo_oca_pos,odoo_repository_org_oca,pos,https://github.com/OCA/pos,https://github.com/OCA/pos,github +repo_oca_product_attribute,odoo_repository_org_oca,product-attribute,https://github.com/OCA/product-attribute,https://github.com/OCA/product-attribute,github +repo_oca_product_configurator,odoo_repository_org_oca,product-configurator,https://github.com/OCA/product-configurator,https://github.com/OCA/product-configurator,github +repo_oca_product_kitting,odoo_repository_org_oca,product-kitting,https://github.com/OCA/product-kitting,https://github.com/OCA/product-kitting,github +repo_oca_product_pack,odoo_repository_org_oca,product-pack,https://github.com/OCA/product-pack,https://github.com/OCA/product-pack,github +repo_oca_product_variant,odoo_repository_org_oca,product-variant,https://github.com/OCA/product-variant,https://github.com/OCA/product-variant,github +repo_oca_program,odoo_repository_org_oca,program,https://github.com/OCA/program,https://github.com/OCA/program,github +repo_oca_project,odoo_repository_org_oca,project,https://github.com/OCA/project,https://github.com/OCA/project,github +repo_oca_project_agile,odoo_repository_org_oca,project-agile,https://github.com/OCA/project-agile,https://github.com/OCA/project-agile,github +repo_oca_project_reporting,odoo_repository_org_oca,project-reporting,https://github.com/OCA/project-reporting,https://github.com/OCA/project-reporting,github +repo_oca_purchase_reporting,odoo_repository_org_oca,purchase-reporting,https://github.com/OCA/purchase-reporting,https://github.com/OCA/purchase-reporting,github +repo_oca_purchase_workflow,odoo_repository_org_oca,purchase-workflow,https://github.com/OCA/purchase-workflow,https://github.com/OCA/purchase-workflow,github +repo_oca_pwa_builder,odoo_repository_org_oca,pwa-builder,https://github.com/OCA/pwa-builder,https://github.com/OCA/pwa-builder,github +repo_oca_queue,odoo_repository_org_oca,queue,https://github.com/OCA/queue,https://github.com/OCA/queue,github +repo_oca_repair,odoo_repository_org_oca,repair,https://github.com/OCA/repair,https://github.com/OCA/repair,github +repo_oca_report_print_send,odoo_repository_org_oca,report-print-send,https://github.com/OCA/report-print-send,https://github.com/OCA/report-print-send,github +repo_oca_reporting_engine,odoo_repository_org_oca,reporting-engine,https://github.com/OCA/reporting-engine,https://github.com/OCA/reporting-engine,github +repo_oca_resource,odoo_repository_org_oca,resource,https://github.com/OCA/resource,https://github.com/OCA/resource,github +repo_oca_rest_api,odoo_repository_org_oca,rest-api,https://github.com/OCA/rest-api,https://github.com/OCA/rest-api,github +repo_oca_rest_framework,odoo_repository_org_oca,rest-framework,https://github.com/OCA/rest-framework,https://github.com/OCA/rest-framework,github +repo_oca_rma,odoo_repository_org_oca,rma,https://github.com/OCA/rma,https://github.com/OCA/rma,github +repo_oca_role_policy,odoo_repository_org_oca,role-policy,https://github.com/OCA/role-policy,https://github.com/OCA/role-policy,github +repo_oca_sale_blanket,odoo_repository_org_oca,sale-blanket,https://github.com/OCA/sale-blanket,https://github.com/OCA/sale-blanket,github +repo_oca_sale_channel,odoo_repository_org_oca,sale-channel,https://github.com/OCA/sale-channel,https://github.com/OCA/sale-channel,github +repo_oca_sale_financial,odoo_repository_org_oca,sale-financial,https://github.com/OCA/sale-financial,https://github.com/OCA/sale-financial,github +repo_oca_sale_promotion,odoo_repository_org_oca,sale-promotion,https://github.com/OCA/sale-promotion,https://github.com/OCA/sale-promotion,github +repo_oca_sale_reporting,odoo_repository_org_oca,sale-reporting,https://github.com/OCA/sale-reporting,https://github.com/OCA/sale-reporting,github +repo_oca_sale_workflow,odoo_repository_org_oca,sale-workflow,https://github.com/OCA/sale-workflow,https://github.com/OCA/sale-workflow,github +repo_oca_search_engine,odoo_repository_org_oca,search-engine,https://github.com/OCA/search-engine,https://github.com/OCA/search-engine,github +repo_oca_server_auth,odoo_repository_org_oca,server-auth,https://github.com/OCA/server-auth,https://github.com/OCA/server-auth,github +repo_oca_server_backend,odoo_repository_org_oca,server-backend,https://github.com/OCA/server-backend,https://github.com/OCA/server-backend,github +repo_oca_server_brand,odoo_repository_org_oca,server-brand,https://github.com/OCA/server-brand,https://github.com/OCA/server-brand,github +repo_oca_server_env,odoo_repository_org_oca,server-env,https://github.com/OCA/server-env,https://github.com/OCA/server-env,github +repo_oca_server_tools,odoo_repository_org_oca,server-tools,https://github.com/OCA/server-tools,https://github.com/OCA/server-tools,github +repo_oca_server_ux,odoo_repository_org_oca,server-ux,https://github.com/OCA/server-ux,https://github.com/OCA/server-ux,github +repo_oca_shift_planning,odoo_repository_org_oca,shift-planning,https://github.com/OCA/shift-planning,https://github.com/OCA/shift-planning,github +repo_oca_shopfloor_app,odoo_repository_org_oca,shopfloor-app,https://github.com/OCA/shopfloor-app,https://github.com/OCA/shopfloor-app +repo_oca_sign,odoo_repository_org_oca,sign,https://github.com/OCA/sign,https://github.com/OCA/sign,github +repo_oca_social,odoo_repository_org_oca,social,https://github.com/OCA/social,https://github.com/OCA/social,github +repo_oca_spreadsheet,odoo_repository_org_oca,spreadsheet,https://github.com/OCA/spreadsheet,https://github.com/OCA/spreadsheet,github +repo_oca_stock_logistics_availability,odoo_repository_org_oca,stock-logistics-availability,https://github.com/OCA/stock-logistics-availability,https://github.com/OCA/stock-logistics-availability,github +repo_oca_stock_logistics_barcode,odoo_repository_org_oca,stock-logistics-barcode,https://github.com/OCA/stock-logistics-barcode,https://github.com/OCA/stock-logistics-barcode,github +repo_oca_stock_logistics_interfaces,odoo_repository_org_oca,stock-logistics-interfaces,https://github.com/OCA/stock-logistics-interfaces,https://github.com/OCA/stock-logistics-interfaces,github +repo_oca_stock_logistics_putaway,odoo_repository_org_oca,stock-logistics-putaway,https://github.com/OCA/stock-logistics-putaway,https://github.com/OCA/stock-logistics-putaway,github +repo_oca_stock_logistics_orderpoint,odoo_repository_org_oca,stock-logistics-orderpoint,https://github.com/OCA/stock-logistics-orderpoint,https://github.com/OCA/stock-logistics-orderpoint,github +repo_oca_stock_logistics_release_channel,odoo_repository_org_oca,stock-logistics-release-channel,https://github.com/OCA/stock-logistics-release-channel,https://github.com/OCA/stock-logistics-release-channel,github +repo_oca_stock_logistics_reporting,odoo_repository_org_oca,stock-logistics-reporting,https://github.com/OCA/stock-logistics-reporting,https://github.com/OCA/stock-logistics-reporting,github +repo_oca_stock_logistics_request,odoo_repository_org_oca,stock-logistics-request,https://github.com/OCA/stock-logistics-request,https://github.com/OCA/stock-logistics-request,github +repo_oca_stock_logistics_reservation,odoo_repository_org_oca,stock-logistics-reservation,https://github.com/OCA/stock-logistics-reservation,https://github.com/OCA/stock-logistics-reservation,github +repo_oca_stock_logistics_shopfloor,odoo_repository_org_oca,stock-logistics-shopfloor,https://github.com/OCA/stock-logistics-shopfloor,https://github.com/OCA/stock-logistics-shopfloor,github +repo_oca_stock_logistics_tracking,odoo_repository_org_oca,stock-logistics-tracking,https://github.com/OCA/stock-logistics-tracking,https://github.com/OCA/stock-logistics-tracking,github +repo_oca_stock_logistics_transport,odoo_repository_org_oca,stock-logistics-transport,https://github.com/OCA/stock-logistics-transport,https://github.com/OCA/stock-logistics-transport,github +repo_oca_stock_logistics_warehouse,odoo_repository_org_oca,stock-logistics-warehouse,https://github.com/OCA/stock-logistics-warehouse,https://github.com/OCA/stock-logistics-warehouse,github +repo_oca_stock_logistics_workflow,odoo_repository_org_oca,stock-logistics-workflow,https://github.com/OCA/stock-logistics-workflow,https://github.com/OCA/stock-logistics-workflow,github +repo_oca_stock_weighing,odoo_repository_org_oca,stock-weighing,https://github.com/OCA/stock-weighing,https://github.com/OCA/stock-weighing,github +repo_oca_storage,odoo_repository_org_oca,storage,https://github.com/OCA/storage,https://github.com/OCA/storage,github +repo_oca_survey,odoo_repository_org_oca,survey,https://github.com/OCA/survey,https://github.com/OCA/survey,github +repo_oca_timesheet,odoo_repository_org_oca,timesheet,https://github.com/OCA/timesheet,https://github.com/OCA/timesheet,github +repo_oca_vertical_abbey,odoo_repository_org_oca,vertical-abbey,https://github.com/OCA/vertical-abbey,https://github.com/OCA/vertical-abbey,github +repo_oca_vertical_agriculture,odoo_repository_org_oca,vertical-agriculture,https://github.com/OCA/vertical-agriculture,https://github.com/OCA/vertical-agriculture,github +repo_oca_vertical_association,odoo_repository_org_oca,vertical-association,https://github.com/OCA/vertical-association,https://github.com/OCA/vertical-association,github +repo_oca_vertical_community,odoo_repository_org_oca,vertical-community,https://github.com/OCA/vertical-community,https://github.com/OCA/vertical-community,github +repo_oca_vertical_construction,odoo_repository_org_oca,vertical-construction,https://github.com/OCA/vertical-construction,https://github.com/OCA/vertical-construction,github +repo_oca_vertical_cooperative_supermarket,odoo_repository_org_oca,vertical-cooperative-supermarket,https://github.com/OCA/vertical-cooperative-supermarket,https://github.com/OCA/vertical-cooperative-supermarket,github +repo_oca_vertical_edition,odoo_repository_org_oca,vertical-edition,https://github.com/OCA/vertical-edition,https://github.com/OCA/vertical-edition,github +repo_oca_vertical_education,odoo_repository_org_oca,vertical-education,https://github.com/OCA/vertical-education,https://github.com/OCA/vertical-education,github +repo_oca_vertical_hotel,odoo_repository_org_oca,vertical-hotel,https://github.com/OCA/vertical-hotel,https://github.com/OCA/vertical-hotel,github +repo_oca_vertical_isp,odoo_repository_org_oca,vertical-isp,https://github.com/OCA/vertical-isp,https://github.com/OCA/vertical-isp,github +repo_oca_vertical_medical,odoo_repository_org_oca,vertical-medical,https://github.com/OCA/vertical-medical,https://github.com/OCA/vertical-medical,github +repo_oca_vertical_ngo,odoo_repository_org_oca,vertical-ngo,https://github.com/OCA/vertical-ngo,https://github.com/OCA/vertical-ngo,github +repo_oca_vertical_realestate,odoo_repository_org_oca,vertical-realestate,https://github.com/OCA/vertical-realestate,https://github.com/OCA/vertical-realestate,github +repo_oca_vertical_rental,odoo_repository_org_oca,vertical-rental,https://github.com/OCA/vertical-rental,https://github.com/OCA/vertical-rental,github +repo_oca_vertical_travel,odoo_repository_org_oca,vertical-travel,https://github.com/OCA/vertical-travel,https://github.com/OCA/vertical-travel,github +repo_oca_wallet,odoo_repository_org_oca,wallet,https://github.com/OCA/wallet,https://github.com/OCA/wallet,github +repo_oca_web,odoo_repository_org_oca,web,https://github.com/OCA/web,https://github.com/OCA/web,github +repo_oca_web_api,odoo_repository_org_oca,web-api,https://github.com/OCA/web-api,https://github.com/OCA/web-api,github +repo_oca_web_api_contrib,odoo_repository_org_oca,web-api-contrib,https://github.com/OCA/web-api-contrib,https://github.com/OCA/web-api-contrib,github +repo_oca_webhook,odoo_repository_org_oca,webhook,https://github.com/OCA/webhook,https://github.com/OCA/webhook,github +repo_oca_webkit_tools,odoo_repository_org_oca,webkit-tools,https://github.com/OCA/webkit-tools,https://github.com/OCA/webkit-tools,github +repo_oca_website,odoo_repository_org_oca,website,https://github.com/OCA/website,https://github.com/OCA/website,github +repo_oca_website_cms,odoo_repository_org_oca,website-cms,https://github.com/OCA/website-cms,https://github.com/OCA/website-cms,github +repo_oca_website_themes,odoo_repository_org_oca,website-themes,https://github.com/OCA/website-themes,https://github.com/OCA/website-themes,github +repo_oca_wms,odoo_repository_org_oca,wms,https://github.com/OCA/wms,https://github.com/OCA/wms,github diff --git a/odoo_repository/data/odoo_branch.xml b/odoo_repository/data/odoo_branch.xml new file mode 100644 index 0000000..23b6fab --- /dev/null +++ b/odoo_repository/data/odoo_branch.xml @@ -0,0 +1,80 @@ + + + + + + + 8.0 + + + + + 9.0 + + + + + 10.0 + + + + + 11.0 + + + + + 12.0 + + + + + 13.0 + + + + + 14.0 + + + + + 15.0 + + + + + 16.0 + + + + + 17.0 + + + + + 18.0 + + + + + + + + + + + + diff --git a/odoo_repository/data/odoo_module.xml b/odoo_repository/data/odoo_module.xml new file mode 100644 index 0000000..bf8ca9c --- /dev/null +++ b/odoo_repository/data/odoo_module.xml @@ -0,0 +1,14 @@ + + + + + server_environment_files + + + + + studio_customization + + + diff --git a/odoo_repository/data/odoo_repository.xml b/odoo_repository/data/odoo_repository.xml new file mode 100644 index 0000000..d6cdfaa --- /dev/null +++ b/odoo_repository/data/odoo_repository.xml @@ -0,0 +1,69 @@ + + + + + + odoo + + https://github.com/odoo/odoo + https://github.com/odoo/odoo + github + + + + + + + enterprise + + + https://github.com/odoo/enterprise + https://github.com/odoo/enterprise + github + + + + + + + design-themes + + https://github.com/odoo/design-themes + https://github.com/odoo/design-themes + github + + + + diff --git a/odoo_repository/data/odoo_repository_addons_path.xml b/odoo_repository/data/odoo_repository_addons_path.xml new file mode 100644 index 0000000..dffbecc --- /dev/null +++ b/odoo_repository/data/odoo_repository_addons_path.xml @@ -0,0 +1,57 @@ + + + + + ./odoo/addons + + + + + ./addons + + + + + . + + + + + . + + + + + + . + + + + + . + + + + + . + + diff --git a/odoo_repository/data/odoo_repository_org.xml b/odoo_repository/data/odoo_repository_org.xml new file mode 100644 index 0000000..be44870 --- /dev/null +++ b/odoo_repository/data/odoo_repository_org.xml @@ -0,0 +1,12 @@ + + + + + odoo + + + + OCA + + diff --git a/odoo_repository/data/queue_job.xml b/odoo_repository/data/queue_job.xml new file mode 100644 index 0000000..f38e67a --- /dev/null +++ b/odoo_repository/data/queue_job.xml @@ -0,0 +1,54 @@ + + + + + odoo_repository_scan + + + + + + _detect_modules_to_scan_on_branch + + + + + + + _scan_module_on_branch + + + + + + + _update_last_scanned_commit + + + + + + odoo_repository_find_pr_url + + + + + + action_find_pr_url + + + + diff --git a/odoo_repository/lib/__init__.py b/odoo_repository/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/odoo_repository/lib/scanner.py b/odoo_repository/lib/scanner.py new file mode 100644 index 0000000..d06dfd6 --- /dev/null +++ b/odoo_repository/lib/scanner.py @@ -0,0 +1,1324 @@ +# Copyright 2023 Camptocamp SA +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl) + +import ast +import contextlib +import json +import logging +import os +import pathlib +import re +import shutil +import tempfile +import time +from urllib.parse import urlparse, urlunparse + +import git +import oca_port +from odoo_addons_parser import ModuleParser + +# Disable logging from 'pygount' (used by odoo_addons_parser) +logging.getLogger("pygount").setLevel(logging.ERROR) + +_logger = logging.getLogger(__name__) + +# Paths ending with these patterns will be ignored such as if all scanned commits +# update such files, the underlying module won't be scanned to preserve resources. +IGNORE_FILES = [".po", ".pot", "README.rst", "index.html"] + +MANIFEST_FILES = ("__manifest__.py", "__openerp__.py") + +AUTHOR_EMAILS_TO_SKIP = [ + "transbot@odoo-community.org", + "noreply@weblate.org", + "oca-git-bot@odoo-community.org", + "oca+oca-travis@odoo-community.org", + "oca-ci@odoo-community.org", + "shopinvader-git-bot@shopinvader.com", +] + +SUMMARY_TERMS_TO_SKIP = [ + "Translated using Weblate", + "Added translation using Weblate", +] + + +@contextlib.contextmanager +def set_env(**environ): + """ + Temporarily set the process environment variables. + + >>> with set_env(PLUGINS_DIR='test/plugins'): + ... "PLUGINS_DIR" in os.environ + True + + >>> "PLUGINS_DIR" in os.environ + False + + :type environ: dict[str, unicode] + :param environ: Environment variables to set + """ + # Copied from: + # https://stackoverflow.com/questions/2059482/ + # temporarily-modify-the-current-processs-environment + old_environ = dict(os.environ) + os.environ.update(environ) + try: + yield + finally: + os.environ.clear() + os.environ.update(old_environ) + + +class BaseScanner: + _dirname = "odoo-repositories" + + def __init__( + self, + org: str, + name: str, + clone_url: str, + branches: list, + repositories_path: str = None, + repo_type: str = None, + ssh_key: str = None, + token: str = None, + workaround_fs_errors: bool = False, + clone_name: str = None, + ): + self.org = org + self.name = name + self.clone_url = self._prepare_clone_url(repo_type, clone_url, token) + self.branches = branches + self.repositories_path = self._prepare_repositories_path(repositories_path) + self.clone_name = clone_name + self.path = self.repositories_path.joinpath( + self.org, self.clone_name or self.name + ) + self.repo_type = repo_type + self.ssh_key = ssh_key + self.token = token + self.workaround_fs_errors = workaround_fs_errors + + def sync(self, fetch=True): + res = True + self._apply_git_global_config() + # Clone or update the repository + if not self.is_cloned: + res = self._clone() + if self.is_cloned: + with self.repo() as repo: + self._apply_git_config(repo) + self._set_git_remote_url(repo, "origin", self.clone_url) + if fetch: + res = self._fetch(repo) + return res + + @contextlib.contextmanager + def _get_git_env(self): + """Context manager yielding env variables used by Git invocations.""" + git_env = {} + if self.ssh_key: + with self._get_ssh_key() as ssh_key_path: + git_ssh_cmd = f"ssh -o StrictHostKeyChecking=no -i {ssh_key_path}" + git_env.update(GIT_SSH_COMMAND=git_ssh_cmd, GIT_TRACE="true") + yield git_env + else: + yield git_env + + @contextlib.contextmanager + def _get_ssh_key(self): + """Save the SSH key in a temporary file and yield its path.""" + with tempfile.NamedTemporaryFile() as fp: + fp.write(self.ssh_key.encode()) + fp.flush() + ssh_key_path = fp.name + yield ssh_key_path + + @staticmethod + def _prepare_clone_url(repo_type, clone_url, token): + """Return the URL used to clone/fetch the repository. + + If a token is provided it will be inserted automatically. + """ + if repo_type in ("github", "gitlab") and token: + parts = list(urlparse(clone_url)) + if parts[0].startswith("http"): + # Update 'netloc' part to prefix it with the OAuth token + parts[1] = f"oauth2:{token}@" + parts[1] + clone_url = urlunparse(parts) + return clone_url + + @classmethod + def _prepare_repositories_path(cls, repositories_path=None): + if not repositories_path: + default_data_dir_path = ( + pathlib.Path.home().joinpath(".local").joinpath("share") + ) + repositories_path = pathlib.Path( + os.environ.get("XDG_DATA_HOME", default_data_dir_path), + cls._dirname, + ) + repositories_path = pathlib.Path(repositories_path) + repositories_path.mkdir(parents=True, exist_ok=True) + return repositories_path + + def _apply_git_global_config(self): + # Avoid 'fatal: detected dubious ownership in repository' errors + # when performing operations in git repositories in case they are + # cloned on an mounted filesystem with specific options. + if self.workaround_fs_errors: + # NOTE: ensure to unset existing entry before adding one, as git doesn't + # check if an entry already exists, generating duplicates + os.system(r"git config --global --unset safe.directory '\*'") + os.system("git config --global --add safe.directory '*'") + + def _apply_git_config(self, repo): + with repo.config_writer() as writer: + # This avoids too high memory consumption (default git config could + # crash the Odoo workers when the scanner is run by Odoo itself). + # This is especially useful to checkout big repositories like odoo/odoo. + writer.set_value("core", "packedGitLimit", "128m") + writer.set_value("core", "packedGitWindowSize", "32m") + writer.set_value("pack", "windowMemory", "64m") + writer.set_value("pack", "threads", "1") + # Avoid issues with file permissions for mounted filesystems + # with specific options. + writer.set_value("core", "filemode", "false") + # Disable some GC features for performance (memory and IO) + # Reflog clean up is triggered automatically by some commands. + # We assume that we scan upstream branches that will never contain + # orphaned commits to clean up, so some GC features are useless in + # this context. + writer.set_value("gc", "pruneExpire", "never") + writer.set_value("gc", "worktreePruneExpire", "never") + writer.set_value("gc", "reflogExpire", "never") + writer.set_value("gc", "reflogExpireUnreachable", "never") + + def _set_git_remote_url(self, repo, remote, url): + """Ensure that `remote` has `url` set.""" + # Check first the URL before setting it, as this triggers a 'chmod' + # command on '.git/config' file (to protect sensitive data) that could + # be not allowed on some mounted file systems. + if remote in repo.remotes: + if repo.remotes[remote].url != url: + repo.remotes[remote].set_url(url) + else: + repo.create_remote(remote, url) + + @property + def is_cloned(self): + return self.path.joinpath(".git").exists() + + @contextlib.contextmanager + def repo(self): + repo = git.Repo(self.path) + try: + yield repo + finally: + del repo + + @property + def full_name(self): + return f"{self.org}/{self.name}" + + def _clone_params(self, **extra): + params = { + "url": self.clone_url, + "to_path": self.path, + "no_checkout": True, + # Avoid issues with file permissions + # "allow_unsafe_options": True, + # "multi_options": ["--config core.filemode=false"], + } + if self.branches: + params["branch"] = self.branches[0] + params.update(extra) + return params + + def _clone(self): + _logger.info("Cloning %s...", self.full_name) + tmp_git_dir_path = None + repo_git_dir_path = pathlib.Path(self.path, ".git") + with tempfile.TemporaryDirectory() as tmp_git_dir: + if self.workaround_fs_errors: + tmp_git_dir_path = pathlib.Path(tmp_git_dir).joinpath(".git") + with self._get_git_env() as git_env: + extra = {"env": git_env} + if self.workaround_fs_errors: + extra["separate_git_dir"] = str(tmp_git_dir_path) + params = self._clone_params(**extra) + try: + git.Repo.clone_from(**params) + except git.exc.GitCommandError as exc: + _logger.error(exc) + if "not found in upstream origin" in str(exc): + _logger.info( + "Couldn't clone remote branch from %s, skipping.", + self.full_name, + ) + return False + else: + if tmp_git_dir_path: + # {repo_path}/.git folder is a hardlink, replace + # it by the .git folder created in /tmp + # NOTE: use shutil instead of 'pathlib.Path.replace()' as + # file systems could be different + if repo_git_dir_path.exists(): + repo_git_dir_path.unlink() + shutil.move(tmp_git_dir_path, repo_git_dir_path) + return True + + def _fetch(self, repo): + _logger.info( + "%s: fetch branch(es) %s", self.full_name, ", ".join(self.branches) + ) + branches_fetched = [] + for branch in self.branches: + # Do not block the process if the branch doesn't exist on this repo + try: + with self._get_git_env() as git_env: + with repo.git.custom_environment(**git_env): + repo.remotes.origin.fetch(branch) + except git.exc.GitCommandError as exc: + _logger.error(exc) + if "couldn't find remote ref" in str(exc): + _logger.info( + "Couldn't find remote branch %s, skipping.", self.full_name + ) + return False + raise + else: + _logger.info("%s: branch %s fetched", self.full_name, branch) + branches_fetched.append(branch) + # Return True as soon as we fetched at least one branch + return bool(branches_fetched) + + def _branch_exists(self, repo, branch, remote="origin"): + refs = [r.name for r in repo.remotes[remote].refs] + branch = f"{remote}/{branch}" + return branch in refs + + def _checkout_branch(self, repo, branch, remote="origin"): + # Ensure to clean up the repository before a checkout + index_lock_path = pathlib.Path(repo.common_dir).joinpath("index.lock") + if index_lock_path.exists(): + index_lock_path.unlink() + repo.git.reset("--hard") + repo.git.clean("-xdf") + repo.git.checkout("-f", f"remotes/{remote}/{branch}") + + def _get_last_fetched_commit(self, repo, branch, remote="origin"): + """Return the last fetched commit for the given `branch`.""" + return repo.rev_parse(f"remotes/{remote}/{branch}").hexsha + + def _get_module_paths(self, repo, relative_path, branch, remote="origin"): + """Return the list of modules available in `branch`.""" + # Clean up 'relative_path' to make it compatible with 'git.Tree' object + relative_tree_path = "/".join( + [dir_ for dir_ in relative_path.split("/") if dir_ and dir_ != "."] + ) + # Return all available modules from 'relative_tree_path' + branch_commit = repo.remotes[remote].refs[branch].commit + addons_trees = branch_commit.tree.trees + if relative_tree_path: + try: + addons_trees = (branch_commit.tree / relative_tree_path).trees + except KeyError: + # 'relative_tree_path' doesn't exist + return [] + module_paths = [tree.path for tree in addons_trees if self._odoo_module(tree)] + return sorted(module_paths) + + def _get_module_paths_updated( + self, + repo, + relative_path, + from_commit, + to_commit, + branch, + ): + """Return modules updated between `from_commit` and `to_commit`. + + It returns a list of modules. + """ + # Clean up 'relative_path' to make it compatible with 'git.Tree' object + relative_tree_path = "/".join( + [dir_ for dir_ in relative_path.split("/") if dir_ and dir_ != "."] + ) + module_paths = set() + # Same commits: nothing has changed + if from_commit == to_commit: + return list(module_paths) + # Get only modules updated between the two commits + from_commit = repo.commit(from_commit) + to_commit = repo.commit(to_commit) + diffs = to_commit.diff(from_commit, R=True) + for diff in diffs: + # Skip diffs that do not belong to the scanned relative path + if not diff.a_path.startswith(relative_tree_path): + continue + # Skip diffs that relates to unrelevant files + if not self._filter_file_path(diff.a_path): + continue + # Exclude files located in root folder + if "/" not in diff.a_path: + continue + # Remove the relative_path (e.g. 'addons/') from the diff path + rel_path = pathlib.Path(relative_path) + diff_path = pathlib.Path(diff.a_path) + module_path = pathlib.Path(*diff_path.parts[: len(rel_path.parts) + 1]) + tree = self._get_subtree(to_commit.tree, str(module_path)) + if tree: + # Module still exists + if self._odoo_module(tree): + module_paths.add(tree.path) + else: + # Module removed + tree = self._get_subtree(from_commit.tree, str(module_path)) + if self._odoo_module(tree): + module_paths.add(tree.path) + return sorted(module_paths) + + def _filter_file_path(self, path): + for ext in (".po", ".pot", ".rst", ".html"): + if path.endswith(ext): + return False + return True + + def _get_last_commit_of_git_tree(self, ref, tree): + return tree.repo.git.log("--pretty=%H", "-n 1", ref, "--", tree.path) + + def _get_commits_of_git_tree(self, from_, to_, tree, patterns=None): + """Returns commits between `from_` and `to_` in chronological order. + + The list of commits can be limited to a `tree`. + """ + if not patterns: + patterns = tuple() + rev_pattern = f"{from_}..{to_}" + if not from_: + rev_pattern = to_ + elif not to_: + rev_pattern = from_ + cmd = [ + "--pretty=%H", + "-r", + rev_pattern, + "--reverse", + "--", + tree.path, + *patterns, + ] + if patterns: + # It's mandatory to use shell here to leverage file patterns + commits = tree.repo.git.execute(" ".join(["git", "log"] + cmd), shell=True) + else: + commits = tree.repo.git.log(cmd) + return commits.split() + + def _odoo_module(self, tree): + """Check if the `git.Tree` object is an Odoo module.""" + # NOTE: it seems we could have data only modules without '__init__.py' + # like 'odoo/addons/test_data_module/', so the Python package check + # is maybe not useful + return self._manifest_exists(tree) # and self._python_package(tree) + + def _python_package(self, tree): + """Check if the `git.Tree` object is a Python package.""" + return bool(self._get_subtree(tree, "__init__.py")) + + def _manifest_exists(self, tree): + """Check if the `git.Tree` object contains an Odoo manifest file.""" + manifest_found = False + for manifest_file in MANIFEST_FILES: + if self._get_subtree(tree, manifest_file): + manifest_found = True + break + return manifest_found + + def _get_subtree(self, tree, path): + """Return the subtree `tree / path` if it exists, or `None`.""" + try: + return tree / path + except KeyError: # pylint: disable=except-pass + pass + + +class MigrationScanner(BaseScanner): + def __init__( + self, + org: str, + name: str, + clone_url: str, + # FIXME: put specific branch names to clone in 'migration_path': + # E.g. [('14.0', 'master'), ('18.0', '18.0-mig')] + migration_path: tuple[str], + new_repo_name: str = None, + new_repo_url: str = None, + repositories_path: str = None, + repo_type: str = None, + ssh_key: str = None, + token: str = None, + workaround_fs_errors: bool = False, + clone_name: str = None, + ): + branches = [mp[1] for mp in sorted(migration_path)] + super().__init__( + org, + name, + clone_url, + branches, + repositories_path, + repo_type, + ssh_key, + token, + workaround_fs_errors, + clone_name, + ) + self.migration_path = migration_path + self.new_repo_name = new_repo_name + self.new_repo_url = ( + self._prepare_clone_url(repo_type, new_repo_url, token) + if new_repo_url + else None + ) + + def sync(self, fetch=True): + res = super().sync(fetch=fetch) + # Set the new repository as remote + if self.is_cloned and self.new_repo_name and self.new_repo_url: + with self.repo() as repo: + self._set_git_remote_url(repo, self.new_repo_name, self.new_repo_url) + return res + + def scan(self, addons_path=".", target_addons_path=".", module_names=None): + # Clone/fetch has been done during the repository scan, the migration + # scan will be processed on the current history of commits, increasing speed. + res = self.sync(fetch=False) + # 'super()' could return False if the branch to scan doesn't exist, + # there is nothing to scan then. + if not res: + return False + source_version = self.migration_path[0][0] + source_branch = self.migration_path[0][1] + target_version = self.migration_path[1][0] + target_branch = self.migration_path[1][1] + target_remote = "origin" + with self.repo() as repo: + if self.new_repo_name and self.new_repo_url: + target_remote = self.new_repo_name + # Fetch target branch from new repo + with self._get_git_env() as git_env: + with repo.git.custom_environment(**git_env): + repo.remotes[target_remote].fetch(target_branch) + if self._branch_exists(repo, source_branch) and self._branch_exists( + repo, target_branch, remote=target_remote + ): + return self._scan_migration_path( + repo, + source_version, + source_branch, + target_remote, + target_version, + target_branch, + addons_path=addons_path, + target_addons_path=target_addons_path, + module_names=module_names, + ) + return res + + def _scan_migration_path( + self, + repo, + source_version, + source_branch, + target_remote, + target_version, + target_branch, + addons_path=".", + target_addons_path=".", + module_names=None, + ): + repo_source_commit = self._get_last_fetched_commit(repo, source_branch) + repo_target_commit = self._get_last_fetched_commit( + repo, target_branch, remote=target_remote + ) + if not module_names: + module_names = self._get_module_paths(repo, addons_path, source_branch) + res = [] + for module in module_names: + if isinstance(module, tuple): + module, target_module = module + else: + target_module = module + if self._is_module_blacklisted(module): + _logger.info( + "%s: '%s' is blacklisted (no migration scan)", + self.full_name, + module, + ) + continue + repo_id = self._get_odoo_repository_id() + module_branch_id = self._get_odoo_module_branch_id( + repo_id, module, source_version + ) + if not module_branch_id: + _logger.warning( + "Module '%(module)s' for version %(version)s does not exist " + "on Odoo, a new scan of the repository is required. Aborted", + {"module": module, "version": source_version}, + ) + continue + # For each module and source/target branch: + # - get commit of 'module' relative to the last fetched commit + # - get commit of 'module' relative to the last scanned commit + module_path = str(pathlib.Path(addons_path).joinpath(module)) + target_module_path = str( + pathlib.Path(target_addons_path).joinpath(target_module) + ) + module_source_tree = self._get_subtree( + repo.commit(repo_source_commit).tree, module_path + ) + # Odoo could know a module that doesn't exist in local repo, this + # could happen if the repo storage has been restored from an older + # backup. In such case, re-fetch the branches. + if not module_source_tree: + _logger.warning( + "%s: module '%s' doesn't exist in branch %s at commit %s, " + "but it could have been added meanwhile. Fetching branches...", + self.full_name, + module, + source_branch, + repo_source_commit, + ) + self.sync() + module_source_tree = self._get_subtree( + repo.commit(repo_source_commit).tree, module_path + ) + if module_source_tree: + _logger.info( + "%s: module '%s' has now been found!", + self.full_name, + module, + ) + module_target_tree = self._get_subtree( + repo.commit(repo_target_commit).tree, target_module_path + ) + module_source_commit = self._get_last_commit_of_git_tree( + repo_source_commit, module_source_tree + ) + module_target_commit = ( + module_target_tree + and self._get_last_commit_of_git_tree( + repo_target_commit, module_target_tree + ) + or False + ) + # Retrieve existing migration data if any and check if it is outdated + data = self._get_odoo_module_branch_migration_data( + repo_id, module, source_version, target_version + ) + if ( + data.get("last_source_mig_scanned_commit") != module_source_commit + or data.get("last_target_mig_scanned_commit") != module_target_commit + ): + scanned_data = self._scan_module( + repo, + addons_path, + target_addons_path, + module, + target_module, + module_branch_id, + source_version, + source_branch, + target_remote, + target_version, + target_branch, + module_source_commit, + module_target_commit, + data.get("last_source_scanned_commit"), + data.get("last_target_scanned_commit"), + data.get("last_source_mig_scanned_commit"), + data.get("last_target_mig_scanned_commit"), + ) + res.append(scanned_data) + return res + + def _scan_module( + self, + repo: git.Repo, + addons_path: str, + target_addons_path: str, + module: str, + target_module: str, + module_branch_id: int, + source_version: str, + source_branch: str, + target_remote: str, + target_version: str, + target_branch: str, + source_commit: str, + target_commit: str, + source_last_scanned_commit: str, + target_last_scanned_commit: str, + source_last_mig_scanned_commit: str, + target_last_mig_scanned_commit: str, + ): + """Collect the migration data of a module.""" + data = { + "addons_path": addons_path, + "target_addons_path": addons_path, + "module": module, + "source_version": source_version, + "source_branch": source_branch, + "target_version": target_version, + "target_branch": target_branch, + "source_commit": source_last_scanned_commit, + "target_commit": target_last_scanned_commit, + } + module_path = str(pathlib.Path(addons_path).joinpath(module)) + target_module_path = str( + pathlib.Path(target_addons_path).joinpath(target_module) + ) + # If files updated in the module since the last scan are not relevant + # (e.g. all new commits are updating PO files), we skip the scan. + source_scan_relevant = self._is_scan_module_relevant( + repo, + module_path, + source_last_mig_scanned_commit, + source_commit, + ) + target_scan_relevant = self._is_scan_module_relevant( + repo, + target_module_path, + target_last_mig_scanned_commit, + target_commit, + ) + # We push the last source/target scanned commits (the ones scanned by + # RepositoryScanner) to Odoo only if a scan is relevant. + # Having the same scanned commit both for code analysis and migration + # stored in Odoo means the migration scan is not needed. + if source_scan_relevant: + data["source_commit"] = source_last_scanned_commit + if target_scan_relevant: + data["target_commit"] = target_last_scanned_commit + scan_relevant = source_scan_relevant or target_scan_relevant + if scan_relevant: + _logger.info( + "%s: relevant changes detected in '%s' (%s -> %s)", + self.full_name, + module if source_scan_relevant else target_module, + source_version, + target_version, + ) + oca_port_data = self._run_oca_port( + module_path, + target_module_path, + source_version, + source_branch, + target_remote, + target_version, + target_branch, + ) + data["report"] = oca_port_data + self._push_scanned_data(module_branch_id, data) + # Mitigate "GH API rate limit exceeds" error + if scan_relevant: + time.sleep(4) + return data + + def _is_scan_module_relevant( + self, + repo: git.Repo, + module_path: str, + last_scanned_commit: str, + last_fetched_commit: str, + ): + """Determine if scanning the module is relevant. + + As the scan of a module can be quite time consuming, we first check + the files impacted among all new commits since the last scan. + If all the files are irrelevants, then we can bypass the scan. + """ + # The first time we want to scan the module obviously + if not last_scanned_commit: + return True + # Module still not available on target branch, no need to re-run a scan + # as it is still "To migrate" in this case + if not last_fetched_commit: + return False + # Other cases: check files impacted by new commits both on source & target + # branches to tell if a scan should be processed + tree = self._get_subtree(repo.commit(last_fetched_commit).tree, module_path) + new_commits = self._get_commits_of_git_tree( + last_scanned_commit, last_fetched_commit, tree + ) + return self._check_relevant_commits(repo, module_path, new_commits) + + def _check_relevant_commits(self, repo, module_path, commits): + paths = set() + for commit_sha in commits: + commit = repo.commit(commit_sha) + if commit.parents: + diffs = commit.diff(commit.parents[0], paths=[module_path], R=True) + else: + diffs = commit.diff(git.NULL_TREE) + for diff in diffs: + paths.add(diff.a_path) + paths.add(diff.b_path) + for path in paths: + if all(not path.endswith(pattern) for pattern in IGNORE_FILES): + return True + return False + + def _run_oca_port( + self, + module_path, + target_module_path, + source_version, + source_branch, + target_remote, + target_version, + target_branch, + ): + _logger.info( + "%s: collect migration data for '%s' (%s -> %s)", + self.full_name, + module_path, + source_branch, + target_branch, + ) + # Initialize the oca-port app + params = { + "source": f"origin/{source_branch}", + "source_version": source_version, + "target": f"{target_remote}/{target_branch}", + "target_version": target_version, + "addon_path": module_path, + "target_addon_path": target_module_path, + "upstream_org": self.org, + "repo_path": self.path, + "repo_name": self.name, + "output": "json", + "fetch": False, + "github_token": self.repo_type == "github" and self.token or None, + } + # Store oca_port cache in the same folder than cloned repositories + # to boost performance of further calls + with set_env(XDG_CACHE_HOME=str(self.repositories_path)): + scan = oca_port.App(**params) + try: + json_data = scan.run() + except ValueError as exc: + _logger.warning(exc) + else: + return json.loads(json_data) + + # Hooks method to override by client class + + def _get_odoo_repository_id(self) -> int: + """Return the ID of the 'odoo.repository' record.""" + raise NotImplementedError + + def _get_odoo_repository_branches(self, repo_id) -> list[str]: + """Return the relevant branches based on 'odoo.repository.branch'.""" + raise NotImplementedError + + def _get_odoo_migration_paths(self, branches) -> list[tuple[str]]: + """Return the available migration paths corresponding to `branches`.""" + raise NotImplementedError + + def _get_odoo_module_branch_id(self, repo_id, module, branch) -> int: + """Return the ID of the 'odoo.module.branch' record.""" + raise NotImplementedError + + def _get_odoo_module_branch_migration_id( + self, module, source_branch, target_branch + ) -> int: + """Return the ID of 'odoo.module.branch.migration' record.""" + raise NotImplementedError + + def _get_odoo_module_branch_migration_data( + self, repo_id, module, source_version, target_version + ) -> dict: + """Return last scanned commits regarding `module`.""" + raise NotImplementedError + + def _push_scanned_data(self, module_branch_id, data): + """Push the scanned module data to Odoo. + + It has to use the 'odoo.module.branch.migration.push_scanned_data' + RPC endpoint. + """ + raise NotImplementedError + + +class RepositoryScanner(BaseScanner): + def __init__( + self, + org: str, + name: str, + clone_url: str, + version: str, + branch: str, + addons_paths_data: list, + repositories_path: str = None, + repo_type: str = None, + ssh_key: str = None, + token: str = None, + workaround_fs_errors: bool = False, + clone_name: str = None, + ): + super().__init__( + org, + name, + clone_url, + [branch], + repositories_path, + repo_type, + ssh_key, + token, + workaround_fs_errors, + clone_name, + ) + self.version = version + self.branch = branch + self.addons_paths_data = addons_paths_data + + def detect_modules_to_scan(self): + res = self.sync() + # 'super()' could return False if the branch to scan doesn't exist, + # there is nothing to scan then. + if not res: + return {} + repo_id = self._get_odoo_repository_id() + with self.repo() as repo: + return self._detect_modules_to_scan(repo, repo_id) + + def _detect_modules_to_scan(self, repo, repo_id): + if not self._branch_exists(repo, self.branch): + return + branch_id = self._get_odoo_branch_id(self.version) + cloned_branch = None + if self.version != self.branch: + cloned_branch = self.branch + repo_branch_id = self._create_odoo_repository_branch( + repo_id, branch_id, cloned_branch=cloned_branch + ) + last_fetched_commit = self._get_last_fetched_commit(repo, self.branch) + last_scanned_commit = self._get_repo_last_scanned_commit(repo_branch_id) + data = { + "repo_branch_id": repo_branch_id, + "last_fetched_commit": last_fetched_commit, + "last_scanned_commit": last_scanned_commit, + "addons_paths": {}, + } + if last_fetched_commit != last_scanned_commit: + # Checkout the source branch to get the last commit of a module working tree + self._checkout_branch(repo, self.branch) + # Scan relevant subfolders of the repository + for addons_path_data in self.addons_paths_data: + addons_path = addons_path_data["relative_path"] + data["addons_paths"][addons_path] = { + "specs": addons_path_data, + "modules_to_scan": self._detect_modules_to_scan_in_addons_path( + repo, + addons_path, + repo_branch_id, + last_fetched_commit, + last_scanned_commit, + ), + } + return data + + def _detect_modules_to_scan_in_addons_path( + self, + repo, + addons_path, + repo_branch_id, + last_fetched_commit, + last_scanned_commit, + ): + if not last_scanned_commit: + # Get all module paths + modules_to_scan = sorted( + self._get_module_paths(repo, addons_path, self.branch) + ) + else: + # Get module paths updated since the last scanned commit + modules_to_scan = self._get_module_paths_updated( + repo, + addons_path, + from_commit=last_scanned_commit, + to_commit=last_fetched_commit, + branch=self.branch, + ) + extra_log = "" + if addons_path != ".": + extra_log = f" in {addons_path}" + _logger.info( + "%s: %s module(s) updated on %s" + extra_log, + self.full_name, + len(modules_to_scan), + self.branch, + ) + return modules_to_scan + + def scan_module(self, module_path, specs): + self._apply_git_global_config() + repo_id = self._get_odoo_repository_id() + branch_id = self._get_odoo_branch_id(self.version) + cloned_branch = None + if self.version != self.branch: + cloned_branch = self.branch + repo_branch_id = self._create_odoo_repository_branch( + repo_id, branch_id, cloned_branch=cloned_branch + ) + with self.repo() as repo: + # Checkout the source branch to perform module code analysis + branch_commit = self._get_last_fetched_commit(repo, self.branch) + if repo.head.commit.hexsha != branch_commit: + self._checkout_branch(repo, self.branch) + # Get last commit of 'module_path' + module_tree = self._get_subtree( + repo.commit(branch_commit).tree, module_path + ) + last_module_commit = ( + self._get_last_commit_of_git_tree(f"{branch_commit}", module_tree) + if module_tree + else None + ) + return self._scan_module( + repo, + repo_branch_id, + module_path, + last_module_commit, + specs, + ) + + def _scan_module( + self, + repo, + repo_branch_id, + module_path, + last_module_commit, + specs, + ): + module = module_path.split("/")[-1] + if self._is_module_blacklisted(module): + _logger.info( + "%s#%s: '%s' is blacklisted (no scan)", + self.full_name, + self.branch, + module_path, + ) + return + last_module_scanned_commit = self._get_module_last_scanned_commit( + repo_branch_id, module + ) + # Do not scan if the module didn't changed since last scan + # NOTE we also do this check at the model level so if the process + # is interrupted (time limit, not enough memory...) we could + # resume the work where it stopped by skipping already scanned + # modules. + if last_module_scanned_commit == last_module_commit: + return + data = {} + if last_module_commit: + _logger.info( + "%s#%s: scan '%s' ", + self.full_name, + self.branch, + module_path, + ) + data = self._run_module_code_analysis( + repo, + module_path, + self.branch, + last_module_scanned_commit, + last_module_commit, + ) + else: + _logger.info( + "%s#%s: '%s' removed", + self.full_name, + self.branch, + module_path, + ) + # Insert all flags 'is_standard', 'is_enterprise', etc + data.update(specs) + # Set the last fetched commit as last scanned commit + data["last_scanned_commit"] = last_module_commit + self._push_scanned_data(repo_branch_id, module, data) + return data + + def _run_module_code_analysis( + self, repo, module_path, branch, from_commit, to_commit + ): + """Perform a code analysis of `module_path`.""" + # Get current code analysis data + parser = ModuleParser(f"{self.path}/{module_path}", scan_models=False) + data = parser.to_dict() + # Append the history of versions + versions = self._read_module_versions( + repo, module_path, branch, from_commit, to_commit + ) + data["versions"] = versions + return data + + def _read_module_versions(self, repo, module_path, branch, from_commit, to_commit): + """Return versions data introduced between `from_commit` and `to_commit`.""" + versions = {} + for manifest_file in MANIFEST_FILES: + manifest_path = "/".join([module_path, manifest_file]) + manifest_tree = self._get_subtree( + repo.commit(to_commit).tree, manifest_path + ) + if not manifest_tree: + continue + new_commits = self._get_commits_of_git_tree( + from_commit, to_commit, manifest_tree + ) + versions_ = self._parse_module_versions_from_commits( + repo, module_path, manifest_path, branch, new_commits + ) + versions.update(versions_) + return versions + + def _parse_module_versions_from_commits( + self, repo, module_path, manifest_path, branch, new_commits + ): + """Parse module versions introduced in `new_commits`.""" + versions = {} + for commit_sha in new_commits: + commit = repo.commit(commit_sha) + if commit.parents: + diffs = commit.diff(commit.parents[0], R=True) + else: + diffs = commit.diff(git.NULL_TREE) + for diff in diffs: + # Check only diffs that update the manifest file + diff_manifest = diff.a_path.endswith( + manifest_path + ) or diff.b_path.endswith(manifest_path) + if not diff_manifest: + continue + # Try to parse the manifest file + try: + manifest_a = ast.literal_eval( + diff.a_blob and diff.a_blob.data_stream.read().decode() or "{}" + ) + manifest_b = ast.literal_eval( + diff.b_blob and diff.b_blob.data_stream.read().decode() or "{}" + ) + except SyntaxError: + _logger.warning(f"Unable to parse {manifest_path} on {branch}") + continue + # Detect version change (added or updated) + if manifest_a.get("version") == manifest_b.get("version"): + continue + if not manifest_b.get("version"): + # Module has been removed? Skipping + continue + version = manifest_b["version"] + # Skip versions that contains special characters + # (often human errors fixed afterwards) + clean_version = re.sub(r"[^0-9\.]", "", version) + if clean_version != version: + continue + # Detect migration script and bind the version to the commit sha + migration_path = "/".join([module_path, "migrations", version]) + migration_tree = self._get_subtree( + repo.tree(f"origin/{branch}"), migration_path + ) + values = { + "commit": commit_sha, + "migration_script": bool(migration_tree), + } + versions[version] = values + return versions + + # Hooks method to override by client class + + def _get_odoo_repository_id(self): + """Return the ID of the 'odoo.repository' record.""" + raise NotImplementedError + + def _get_odoo_branch_id(self, version): + """Return the ID of the relevant 'odoo.branch' record.""" + raise NotImplementedError + + def _get_odoo_repository_branch_id(self, repo_id, branch_id): + """Return the ID of the 'odoo.repository.branch' record.""" + raise NotImplementedError + + def _create_odoo_repository_branch(self, repo_id, branch_id): + """Create an 'odoo.repository.branch' record and return its ID.""" + raise NotImplementedError + + def _get_repo_last_scanned_commit(self, repo_branch_id): + """Return the last scanned commit of the repository/branch.""" + raise NotImplementedError + + def _is_module_blacklisted(self, module): + """Check if `module` is blacklisted (and should not be scanned).""" + raise NotImplementedError + + def _get_module_last_scanned_commit(self, repo_branch_id, module): + """Return the last scanned commit of the module.""" + raise NotImplementedError + + def _push_scanned_data(self, repo_branch_id, module, data): + """Push the scanned module data to Odoo. + + It has to use the 'odoo.module.branch.push_scanned_data' RPC endpoint. + """ + raise NotImplementedError + + def _update_last_scanned_commit(self, repo_branch_id, last_scanned_commit): + """Update the last scanned commit for the repository/branch.""" + raise NotImplementedError + + +class ChangelogScanner(BaseScanner): + """Generate a changelog for a repository used in a project.""" + + def __init__( + self, + org: str, + name: str, + clone_url: str, + odoo_project_repository_id: int, + repositories_path: str = None, + repo_type: str = None, + ssh_key: str = None, + token: str = None, + ): + self.odoo_project_repository_id = odoo_project_repository_id + data = self._get_odoo_project_repository_data(odoo_project_repository_id) + self.branch = data["branch"] + self.source_commit = data["source_commit"] + self.target_commit = data["target_commit"] or f"origin/{self.branch}" + self.modules = data["modules"] + super().__init__( + org, + name, + clone_url, + [self.branch], + repositories_path, + repo_type, + ssh_key, + token, + ) + + def scan(self): + res = self.sync() + changelog = self._generate_changelog() + self._push_odoo_project_repository_changelog( + self.odoo_project_repository_id, changelog + ) + return res + + def _generate_changelog(self): + with self.repo() as repo: + if not self._branch_exists(repo, self.branch): + return + last_commit = self._get_last_fetched_commit(repo, self.branch) + changelog = { + "source_commit": self.source_commit, + "target_commit": last_commit, + "modules": {}, + } + for module_data in self.modules: + module_path = module_data["path"] + _logger.info( + "%s#%s: generate changelog for %s", + self.full_name, + self.branch, + module_path, + ) + module_changelog = self._generate_module_changelog(repo, module_path) + if module_changelog: + changelog["modules"][module_data["id"]] = module_changelog + return changelog + + def _generate_module_changelog(self, repo, module_path): + changelog = [] + tree = self._get_subtree(repo.commit(self.source_commit).tree, module_path) + if not tree: + return changelog + # Leverage git pathspecs magic (patterns) as it is faster than checking + # the content (diffs) within Python process to get only relevant commits.. + commits = self._get_commits_of_git_tree( + self.source_commit, + self.target_commit, + tree, + patterns=( + "':^*/i18n/*'", + "':^*/i18n_extra/*'", + "':^*.html'", + "':^*.rst'", + "':^*/tests/*'", + "':^*/demo/*'", + "':^*/doc/*'", + ), + ) + for commit_sha in commits: + commit = repo.commit(commit_sha) + if self._skip_commit(commit): + continue + changelog.append(self._prepare_module_changelog(commit)) + return changelog + + @staticmethod + def _skip_commit(commit): + """Check if a commit should be skipped or not. + + E.g merge or translations commits are skipped. + """ + return ( + # Skip merge commit + len(commit.parents) > 1 + or commit.author.email in AUTHOR_EMAILS_TO_SKIP + or any([term in commit.summary for term in SUMMARY_TERMS_TO_SKIP]) + ) + + def _prepare_module_changelog(self, commit): + message = commit.message.split("\n") + message.pop(0) # Remove redundant summary (first line) + message = "\n".join(message).strip() + return { + "hexsha": commit.hexsha, + "authored_datetime": commit.authored_datetime.replace( + tzinfo=None + ).isoformat(), + "summary": commit.summary, + "message": message, + } + + def _get_odoo_project_repository_data(self, project_repo_id): + """Return required data to generate the changelog. + + Return a dictionary such as: + + { + "odoo_project_id": 10, + "branch": "17.0", + "source_commit": "7b58a288b3d79fbdc91dbf14aaeac0d69d65c327", + "target_commit": None, + "modules": [ + # List of dicts {"id": PROJECT_MODULE_ID, ...} + {"id": 1, "name": "base", "path": "odoo/addons/base"}, + {"id": 2, "name": "account", "path": "addons/account"}, + ] + } + """ + raise NotImplementedError + + def _push_odoo_project_repository_changelog(self, project_repo_id, changelog): + """Push the resulting changelog to its 'odoo.project.repository' object. + + It has to use the 'odoo.project.repository.push_changelog' RPC endpoint. + """ + raise NotImplementedError diff --git a/odoo_repository/models/__init__.py b/odoo_repository/models/__init__.py new file mode 100644 index 0000000..e32af68 --- /dev/null +++ b/odoo_repository/models/__init__.py @@ -0,0 +1,18 @@ +from . import authentication_token +from . import ssh_key +from . import odoo_author +from . import odoo_branch +from . import odoo_license +from . import odoo_maintainer +from . import odoo_module +from . import odoo_module_category +from . import odoo_module_dev_status +from . import odoo_python_dependency +from . import odoo_repository_org +from . import odoo_repository_addons_path +from . import odoo_repository +from . import odoo_repository_branch +from . import odoo_module_branch +from . import odoo_module_branch_version +from . import res_company +from . import res_config_settings diff --git a/odoo_repository/models/authentication_token.py b/odoo_repository/models/authentication_token.py new file mode 100644 index 0000000..754fa58 --- /dev/null +++ b/odoo_repository/models/authentication_token.py @@ -0,0 +1,12 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class AuthenticationToken(models.Model): + _name = "authentication.token" + _description = "Authentication Token" + + name = fields.Char(required=True) + token = fields.Char(required=True) diff --git a/odoo_repository/models/odoo_author.py b/odoo_repository/models/odoo_author.py new file mode 100644 index 0000000..d2eeb8e --- /dev/null +++ b/odoo_repository/models/odoo_author.py @@ -0,0 +1,16 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class OdooAuthor(models.Model): + _name = "odoo.author" + _description = "Odoo Module Author" + _order = "name" + + name = fields.Char(required=True, index=True) + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This author already exists."), + ] diff --git a/odoo_repository/models/odoo_branch.py b/odoo_repository/models/odoo_branch.py new file mode 100644 index 0000000..31af62c --- /dev/null +++ b/odoo_repository/models/odoo_branch.py @@ -0,0 +1,111 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import re + +from odoo import _, api, fields, models +from odoo.exceptions import ValidationError + + +class OdooBranch(models.Model): + _name = "odoo.branch" + _description = "Odoo Branch/Version" + _order = "sequence, name" + + name = fields.Char( + string="Version", + required=True, + index=True, + help=( + "An Odoo version is also used as an Odoo branch name in generic " + "repositories (Odoo, OCA...)." + ), + ) + active = fields.Boolean(default=True) + repository_branch_ids = fields.One2many( + comodel_name="odoo.repository.branch", + inverse_name="branch_id", + string="Repositories", + readonly=True, + ) + sequence = fields.Integer() + next_id = fields.Many2one( + comodel_name="odoo.branch", + compute="_compute_next_id", + ) + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This branch already exists."), + ] + + @api.constrains("name") + def _constrains_name(self): + odoo_version_pattern = r"^[0-9]+\.[0-9]$" + for rec in self: + version = re.search(odoo_version_pattern, rec.name) + if not version: + raise ValidationError(_("Version must match the pattern 'x.y'.")) + + @api.depends("sequence") + def _compute_next_id(self): + for rec in self: + rec.next_id = self.search( + [("sequence", ">", rec.sequence)], + order="sequence", + limit=1, + ) + + @api.model + def _recompute_sequence(self): + """Recompute the 'sequence' field to get release branches sorted.""" + self.flush_recordset() + odoo_versions_to_recompute = self._get_all_odoo_versions() + for odoo_version in odoo_versions_to_recompute: + query = """ + UPDATE odoo_branch + SET sequence = ( + SELECT pos.position + FROM ( + SELECT + id, + row_number() OVER ( + ORDER BY string_to_array(name, '.')::int[] + ) AS position + FROM odoo_branch + ) as pos + WHERE pos.id = %(id)s + ) + WHERE id = %(id)s; + """ + args = { + "id": odoo_version.id, + } + self.env.cr.execute(query, args) + self.invalidate_recordset(["sequence"]) + + @api.model_create_multi + def create(self, vals_list): + res = super().create(vals_list) + res._recompute_sequence() + return res + + def write(self, values): + res = super().write(values) + self._recompute_sequence() + return res + + def action_scan(self, force=False): + """Scan this branch in all repositories.""" + self.repository_branch_ids.action_scan(force=force) + + def action_force_scan(self): + """Force the scan of this branch in all repositories. + + It will restart the scan without considering the last scanned commit, + overriding already collected module data if any. + """ + return self.action_scan(force=True) + + def _get_all_odoo_versions(self, active_test=False): + """Return all Odoo versions, even archived ones.""" + return self.with_context(active_test=active_test).search([]) diff --git a/odoo_repository/models/odoo_license.py b/odoo_repository/models/odoo_license.py new file mode 100644 index 0000000..d605686 --- /dev/null +++ b/odoo_repository/models/odoo_license.py @@ -0,0 +1,16 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class OdooLicense(models.Model): + _name = "odoo.license" + _description = "Odoo License" + _order = "name" + + name = fields.Char(required=True, index=True) + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This license already exists."), + ] diff --git a/odoo_repository/models/odoo_maintainer.py b/odoo_repository/models/odoo_maintainer.py new file mode 100644 index 0000000..2367369 --- /dev/null +++ b/odoo_repository/models/odoo_maintainer.py @@ -0,0 +1,31 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import api, fields, models + +from ..utils.github import GITHUB_URL + + +class OdooMaintainer(models.Model): + _name = "odoo.maintainer" + _description = "Odoo Module Maintainer" + _order = "name" + + name = fields.Char(required=True, index=True) + github_url = fields.Char(string="GitHub URL", compute="_compute_github_url") + module_branch_ids = fields.Many2many( + comodel_name="odoo.module.branch", + relation="module_branch_maintainer_rel", + column1="maintainer_id", + column2="module_branch_id", + string="Maintainers", + ) + + @api.depends("name") + def _compute_github_url(self): + for rec in self: + rec.github_url = f"{GITHUB_URL}/{rec.name}" + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This maintainer already exists."), + ] diff --git a/odoo_repository/models/odoo_module.py b/odoo_repository/models/odoo_module.py new file mode 100644 index 0000000..24c8ba9 --- /dev/null +++ b/odoo_repository/models/odoo_module.py @@ -0,0 +1,24 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class OdooModule(models.Model): + _name = "odoo.module" + _description = "Odoo Module Technical Name" + + name = fields.Char(required=True, index=True, help="Technical Name") + module_branch_ids = fields.One2many( + comodel_name="odoo.module.branch", + inverse_name="module_id", + string="Modules", + readonly=True, + ) + blacklisted = fields.Boolean( + help="Blacklisted modules won't be scanned.", readonly=True + ) + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This module technical name already exists."), + ] diff --git a/odoo_repository/models/odoo_module_branch.py b/odoo_repository/models/odoo_module_branch.py new file mode 100644 index 0000000..ba39bff --- /dev/null +++ b/odoo_repository/models/odoo_module_branch.py @@ -0,0 +1,905 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import pathlib +import random +import time +from urllib.parse import urlparse + +from odoo import _, api, fields, models, tools +from odoo.exceptions import ValidationError +from odoo.osv import expression + +from odoo.addons.queue_job.exception import RetryableJobError + +from ..utils import github +from ..utils.module import adapt_version + + +class OdooModuleBranch(models.Model): + _name = "odoo.module.branch" + _description = "Odoo Module Branch" + _order = "repository_sequence, module_name, branch_name" + + module_id = fields.Many2one( + comodel_name="odoo.module", + ondelete="restrict", + string="Technical name", + required=True, + index=True, + ) + module_name = fields.Char( + string="Module Technical Name", related="module_id.name", store=True, index=True + ) + repository_branch_id = fields.Many2one( + comodel_name="odoo.repository.branch", + ondelete="set null", + string="Repository Branch", + index=True, + ) + repository_id = fields.Many2one( + related="repository_branch_id.repository_id", + store=True, + index=True, + precompute=True, + string="Repository", + ) + repository_sequence = fields.Integer( + related="repository_id.sequence", + store=True, + index=True, + ) + org_id = fields.Many2one( + related="repository_branch_id.repository_id.org_id", + store=True, + string="Organization", + ) + branch_id = fields.Many2one( + # NOTE: not a related on 'repository_branch_id' as we need to create + # modules without knowing in advance what is their repo (orphaned modules). + comodel_name="odoo.branch", + ondelete="cascade", + string="Odoo Version", + required=True, + index=True, + ) + branch_name = fields.Char( + string="Branch Name", related="branch_id.name", store=True, index=True + ) + branch_sequence = fields.Integer( + string="Branch Sequence", related="branch_id.sequence", store=True, index=True + ) + pr_url = fields.Char(string="PR URL") + is_standard = fields.Boolean( + string="Standard?", + help="Is this module part of Odoo standard?", + default=False, + ) + is_enterprise = fields.Boolean( + string="Enterprise?", + help="Is this module designed for Odoo Enterprise only?", + default=False, + ) + is_community = fields.Boolean( + string="Community?", + help="Is this module a contribution of the community?", + default=False, + ) + title = fields.Char(index=True, help="Descriptive name") + name = fields.Char( + string="Techname", + compute="_compute_name", + store=True, + index=True, + ) + summary = fields.Char(index=True) + category_id = fields.Many2one( + comodel_name="odoo.module.category", + ondelete="restrict", + string="Category", + index=True, + ) + author_ids = fields.Many2many( + comodel_name="odoo.author", + string="Authors", + ) + maintainer_ids = fields.Many2many( + comodel_name="odoo.maintainer", + relation="module_branch_maintainer_rel", + column1="module_branch_id", + column2="maintainer_id", + string="Maintainers", + ) + dependency_ids = fields.Many2many( + comodel_name="odoo.module.branch", + relation="module_branch_dependency_rel", + column1="module_branch_id", + column2="dependency_id", + string="Dependencies", + ) + reverse_dependency_ids = fields.Many2many( + comodel_name="odoo.module.branch", + relation="module_branch_dependency_rel", + column1="dependency_id", + column2="module_branch_id", + string="Reverse Dependencies", + ) + global_dependency_level = fields.Integer( + compute="_compute_dependency_level", + recursive=True, + store=True, + string="Global Dep. Level", + help="Dependency level including all standard Odoo modules.", + ) + non_std_dependency_level = fields.Integer( + compute="_compute_dependency_level", + recursive=True, + store=True, + string="Non-Std Dep. Level", + help="Dependency level excluding all standard Odoo modules.", + ) + license_id = fields.Many2one( + comodel_name="odoo.license", + ondelete="restrict", + string="License", + index=True, + ) + version = fields.Char("Last version") + version_ids = fields.One2many( + comodel_name="odoo.module.branch.version", + inverse_name="module_branch_id", + string="Versions", + ) + development_status_id = fields.Many2one( + comodel_name="odoo.module.dev.status", + ondelete="restrict", + string="Develoment Status", + index=True, + ) + external_dependencies = fields.Serialized() + python_dependency_ids = fields.Many2many( + comodel_name="odoo.python.dependency", + string="Python Dependencies", + ) + application = fields.Boolean(default=False) + installable = fields.Boolean(default=True) + auto_install = fields.Boolean( + string="Auto-Install", + default=False, + ) + sloc_python = fields.Integer("Python", help="Python source lines of code") + sloc_xml = fields.Integer("XML", help="XML source lines of code") + sloc_js = fields.Integer("JS", help="JavaScript source lines of code") + sloc_css = fields.Integer("CSS", help="CSS source lines of code") + last_scanned_commit = fields.Char() + removed = fields.Boolean() + addons_path = fields.Char( + help="Technical field. Where the module is located in the repository." + ) + full_path = fields.Char(compute="_compute_full_path") + url = fields.Char("URL", compute="_compute_url") + specific = fields.Boolean( + help=( + "Module specific to a project repository." + "It cannot be used across different projects." + ) + ) + + _sql_constraints = [ + ( + "module_id_branch_id_repository_id_uniq", + "UNIQUE (module_id, branch_id, repository_id)", + "This module already exists for this repository/branch.", + ), + ] + + def init(self): + # Index to complete unique constraint 'module_id_branch_id_repository_id_uniq'. + # This is mandatory to support repository_id=NULL in the constraint, + # so we cannot create the same orphaned module twice. + indexes = [ + # PostgreSQL < 15 (partial indexes) + """ + CREATE UNIQUE INDEX IF NOT EXISTS odoo_module_branch_uniq_not_null + ON odoo_module_branch (module_id, branch_id, repository_id) + WHERE repository_id IS NOT NULL; + """, + """ + CREATE UNIQUE INDEX IF NOT EXISTS odoo_module_branch_uniq_null + ON odoo_module_branch (module_id, branch_id) + WHERE repository_id IS NULL; + """, + # PostgreSQL >= 15 (with NULLS NOT DISTINCT) + # """ + # CREATE UNIQUE INDEX odoo_module_branch_uniq + # ON odoo_module_branch (module_id, branch_id, repository_id) + # NULLS NOT DISTINCT; + # """ + ] + for index in indexes: + self._cr.execute(index) + + @api.constrains("specific", "dependency_ids") + def _check_generic_depends_on_specific(self): + for rec in self: + if not rec.specific: + specific_deps = rec.dependency_ids.filtered("specific") + if specific_deps: + msg = _( + "Generic module %(generic_mod)s cannot depend " + "on specific module(s): %(specific_mods)s" + ) + raise ValidationError( + msg + % { + "generic_mod": rec.module_name, + "specific_mods": ", ".join( + specific_deps.mapped("module_name") + ), + } + ) + + @api.depends("module_name", "addons_path") + def _compute_full_path(self): + for rec in self: + rec.full_path = pathlib.Path(rec.addons_path or ".").joinpath( + rec.module_name + ) + + @api.depends( + "repository_id.repo_url", + "branch_name", + "repository_branch_id.cloned_branch", + "addons_path", + "module_name", + ) + def _compute_url(self): + for rec in self: + rec.url = False + if not rec.repository_id: + continue + branch = rec.branch_name + if rec.repository_branch_id.cloned_branch: + branch = rec.repository_branch_id.cloned_branch + module_path = "/".join([rec.addons_path or ".", rec.module_name]) + rec.url = rec.repository_id._get_resource_url(branch, module_path) + + @api.depends("repository_branch_id.name", "module_id.name") + def _compute_name(self): + for rec in self: + rec.name = f"{rec.repository_branch_id.name or '?'} - {rec.module_id.name}" + + @api.depends( + "dependency_ids.global_dependency_level", + "dependency_ids.non_std_dependency_level", + "dependency_ids.is_standard", + ) + def _compute_dependency_level(self): + for rec in self: + global_max_parent_level = max( + [dep.global_dependency_level for dep in rec.dependency_ids] + [0] + ) + rec.global_dependency_level = global_max_parent_level + 1 + non_std_max_parent_level = max( + [ + dep.non_std_dependency_level + for dep in rec.dependency_ids + if not dep.is_standard + ] + + [0] + ) + rec.non_std_dependency_level = ( + # Set 0 on all std modules so they will always have a dependency + # level inferior to non-std modules + (non_std_max_parent_level + 1) if not rec.is_standard else 0 + ) + + def _get_recursive_dependencies(self, domain=None): + """Return all dependencies recursively. + + A domain can be applied to restrict the modules to return, e.g: + + >>> mod._get_recursive_dependencies([("org_id", "=", "OCA")]) + + """ + if not domain: + domain = [] + dependencies = self.dependency_ids.filtered_domain(domain) + dep_ids = set(dependencies.ids) + for dep in dependencies: + dep_ids |= set( + dep._get_recursive_dependencies().filtered_domain(domain).ids + ) + return self.browse(dep_ids) + + def open_recursive_dependencies(self): + self.ensure_one() + xml_id = "odoo_repository.odoo_module_branch_action_recursive_dependencies" + action = self.env["ir.actions.actions"]._for_xml_id(xml_id) + action["name"] = "All dependencies" + action["domain"] = [("id", "in", self._get_recursive_dependencies().ids)] + return action + + def action_find_pr_url(self): + """Find the PR on GitHub that adds this module.""" + self.ensure_one() + if self.pr_url or self.repository_branch_id or self.specific: + return False + values = {"pr_url": False} + pr_urls = self._find_pr_urls_from_github(self.branch_id, self.module_id) + for pr_url in pr_urls: + values["pr_url"] = pr_url + # Get the relevant repository from PR URL if not yet defined + if not self.repository_branch_id: + repository = self._find_repository_from_pr_url(pr_url) + if not repository: + continue + repository_branch = self.env["odoo.repository.branch"].search( + [ + ("repository_id", "=", repository.id), + ("branch_id", "=", self.branch_id.id), + ] + ) + if repository_branch: + values["repository_branch_id"] = repository_branch.id + break + self.sudo().write(values) + return True + + def _find_pr_urls_from_github(self, branch, module): + """Find the GitHub Pull Requests adding `module` on `branch`.""" + # Look for an open PR first, then unmerged (which includes closed ones) + for pr_state in ("open", "unmerged"): + url = ( + f"search/issues?q=is:pr+is:{pr_state}+base:{branch.name}" + f"+in:title+{module.name}" + ) + try: + # Mitigate 'API rate limit exceeded' GitHub API error + # by adding a random waiting time of 1-4s + time.sleep(random.randrange(1, 5)) + prs = github.request(self.env, url) + except RuntimeError as exc: + raise RetryableJobError("Error while looking for PR URL") from exc + for pr in prs.get("items", []): + yield pr["html_url"] + + def _find_repository_from_pr_url(self, pr_url): + """Return the repository corresponding to `pr_url`.""" + # Extract organization and repository name from PR url + path_parts = list(filter(None, urlparse(pr_url).path.split("/"))) + org, repository = path_parts[:2] + repository_model = self.env["odoo.repository"].with_context(active_test=False) + return repository_model.search( + [ + ("org_id", "=", org), + ("name", "=", repository), + ] + ) + + @api.model + @api.returns("odoo.module.branch") + def push_scanned_data(self, repo_branch_id, module, data): + """Entry point for the scanner to push its data.""" + module = self._get_module(module) + repo_branch = self.env["odoo.repository.branch"].browse(repo_branch_id) + values = self._prepare_module_branch_values(repo_branch, module, data) + return self._create_or_update(repo_branch, module, values) + + def _prepare_module_branch_values(self, repo_branch, module, data): + # Get existing module.branch (hosted in scanned repo) if any + module_branch = self._get_module_branch( + repo_branch.branch_id, module, repo=repo_branch.repository_id + ) + # Prepare the 'odoo.module.branch' values + manifest = data.get("manifest", {}) + values = { + "repository_branch_id": repo_branch.id, + "branch_id": repo_branch.branch_id.id, + "module_id": module.id, + "is_standard": data["is_standard"], + "is_enterprise": data["is_enterprise"], + "is_community": data["is_community"], + "last_scanned_commit": data.get("last_scanned_commit", False), + "addons_path": data["relative_path"], + "specific": repo_branch.repository_id.specific, + # Unset PR URL once the module is available in the repository. + "pr_url": False, + } + if manifest: + category_id = self._get_module_category_id(manifest.get("category", "")) + author_ids = self._get_author_ids(manifest.get("author", "")) + maintainer_ids = self._get_maintainer_ids( + tuple(manifest.get("maintainers", [])) + ) + dev_status_id = self._get_dev_status_id( + manifest.get("development_status", "") + ) + dependency_ids = [] + external_dependencies = {} + python_dependency_ids = [] + if manifest.get("installable", True): + dependency_ids = self._get_dependency_ids( + repo_branch, + # Set at least a dependency on "base" if not defined + manifest.get("depends") or ["base"], + ) + external_dependencies = manifest.get("external_dependencies", {}) + python_dependency_ids = self._get_python_dependency_ids( + tuple(external_dependencies.get("python", [])) + ) + license_id = self._get_license_id(manifest.get("license", "")) + values.update( + { + "title": manifest.get("name", False), + "summary": manifest.get( + "summary", manifest.get("description", False) + ), + "category_id": category_id, + "author_ids": [(6, 0, author_ids)], + "maintainer_ids": [(6, 0, maintainer_ids)], + "dependency_ids": [(6, 0, dependency_ids)], + "external_dependencies": external_dependencies, + "python_dependency_ids": [(6, 0, python_dependency_ids)], + "license_id": license_id, + "version": manifest.get("version", False), + "development_status_id": dev_status_id, + "application": manifest.get("application", False), + "installable": manifest.get("installable", True), + "auto_install": manifest.get("auto_install", False), + } + ) + if data.get("last_scanned_commit"): + values.update( + { + "removed": False, + "sloc_python": data["code"]["Python"], + "sloc_xml": data["code"]["XML"], + "sloc_js": data["code"]["JavaScript"], + "sloc_css": data["code"]["CSS"], + } + ) + # Handle module removal + elif module_branch: + values.update( + { + "installable": False, + "removed": True, + } + ) + # Handle versions history + if values.get("installable"): + versions = self._prepare_module_branch_version_ids_values( + repo_branch, + module_branch, + module, + # If no history versions was scanned (could happen if versions are + # part of an unfetched branch), create one corresponding to the + # current manifest version if any but without commit. + versions=( + data.get("versions") + or ( + {values["version"]: {"commit": None}} + if values.get("version") + else {} + ) + ), + ) + if versions: + values["version_ids"] = versions + return values + + def _create_or_update(self, repo_branch, module, values): + """Create or update a `odoo.module.branch` record from scanned module. + + This method will try to link/update an existing module in DB, that could be: + - already scanned in the current repository (simple update) + - orphaned (update the repository of such module) + - unmerged/pending (only if the scanned repository hosts generic modules) + """ + branch = repo_branch.branch_id + module_branch = False + module_branch_in_repo = self._get_module_branch( + branch, module, repo=repo_branch.repository_id + ) + # Module was already scanned in the current repository: update it + if module_branch_in_repo: + module_branch = module_branch_in_repo + # Module was never scanned in the current repository: + else: + # Check if an orphaned module exists + orphaned_module_branch = self._get_orphaned_module_branch(branch, module) + if orphaned_module_branch: + module_branch = orphaned_module_branch + # Check if an unmerged module exists if the scanned repo is generic + elif not repo_branch.repository_id.specific: + unmerged_module_branch = self._get_unmerged_module_branch( + branch, module + ) + if unmerged_module_branch: + module_branch = unmerged_module_branch + module_branch = self._filter_module_to_update(repo_branch, module_branch) + if module_branch: + values["repository_branch_id"] = repo_branch.id + module_branch.sudo().write(values) + else: + module_branch = self.sudo().create(values) + # Special case: when creating 'base' module, ensure that all previously + # scanned modules without dependency for the same Odoo version get + # a dependency against this 'base' module. + if module_branch.module_name == "base": + module_branch._update_modules_to_depend_on_base() + return module_branch + + @api.model + def _update_modules_to_depend_on_base(self): + """Make all scanned modules without dependency depending on 'base'. + + It is executed when a 'base' module is scanned for the first time. + """ + # Update only scanned modules (ones found in repositories) + all_modules = self.search( + [ + ("dependency_ids", "=", False), + ("last_scanned_commit", "!=", False), + ("branch_id", "!=", False), + ("module_name", "!=", "base"), + ], + ) + for branch, modules in tools.groupby(all_modules, key=lambda m: m.branch_id): + base = self.search( + [("module_name", "=", "base"), ("branch_id", "=", branch.id)], + limit=1, + ) + if not base: + continue + self.union(*modules).dependency_ids |= base + + def _filter_module_to_update(self, repo_branch, module_branch): + """Hook called by '_create_or_update'. + + Can be overriden to return `False` to force the creation of a new + `odoo.module.branch` record linked to the scanned repository. + """ + return module_branch + + @api.model + def _get_existing_version(self, module, manifest_value, commit): + if not commit: + return self.env["odoo.module.branch.version"] + return self.env["odoo.module.branch.version"].search( + [ + ("module_name", "=", module.name), + ("manifest_value", "=", manifest_value), + ("commit", "=", commit), + ], + limit=1, + ) + + def _prepare_module_branch_version_ids_values( + self, repo_branch, module_branch, module, versions + ): + # Insert new versions + version_ids = [] + other_odoo_versions = ( + self.env["odoo.branch"]._get_all_odoo_versions() - repo_branch.branch_id + ) + for manifest_value, data in versions.items(): + # Version scanned doesn't belong to the current branch, skipping + if any( + manifest_value.startswith(odoo_version.name + ".") + for odoo_version in other_odoo_versions + ): + continue + name = adapt_version(repo_branch.branch_id.name, manifest_value) + # As we could import versions history from previous Odoo releases + # (i.e. the branch has been started from a previous one), check if + # it hasn't been imported already thanks to the related commit SHA + version = self._get_existing_version(module, manifest_value, data["commit"]) + if version and module_branch: + # Skip if the version has already been imported for a + # previous Odoo release + if version.branch_id.sequence < module_branch.branch_id.sequence: + continue + # Corner case: we scanned a version that was already imported + # through a newer Odoo branch. Downgrade the existing version + # to the current module branch. + if version.branch_id.sequence > module_branch.branch_id.sequence: + version.write( + { + "module_branch_id": module_branch.id, + "name": name, + } + ) + continue + module_version = module_branch.version_ids.filtered( + lambda v, name=name, manifest_value=manifest_value: ( + v.name == name and v.manifest_value == manifest_value + ) + ) + values = { + "name": name, + "manifest_value": manifest_value, + "commit": data["commit"], + "has_migration_script": data.get("migration_script", False), + } + if module_version: + version_ids.append(fields.Command.update(module_version.id, values)) + else: + version_ids.append(fields.Command.create(values)) + return version_ids + + @tools.ormcache("category_name") + def _get_module_category_id(self, category_name): + if category_name: + rec = self.env["odoo.module.category"].search( + [("name", "=", category_name)], limit=1 + ) + if not rec: + rec = ( + self.env["odoo.module.category"] + .sudo() + .create({"name": category_name}) + ) + return rec.id + return False + + @tools.ormcache("names") + def _get_author_ids(self, names): + if names: + # Some Odoo std modules have a list instead of a string as 'author' + if isinstance(names, str): + names = [name.strip() for name in names.split(",")] + authors = self.env["odoo.author"].search([("name", "in", names)]) + missing_author_names = set(names) - set(authors.mapped("name")) + missing_authors = self.env["odoo.author"] + if missing_author_names: + missing_authors = ( + self.env["odoo.author"] + .sudo() + .create([{"name": name} for name in missing_author_names]) + ) + return (authors | missing_authors).ids + return [] + + @tools.ormcache("names") + def _get_maintainer_ids(self, names): + if names: + maintainers = self.env["odoo.maintainer"].search([("name", "in", names)]) + missing_maintainer_names = set(names) - set(maintainers.mapped("name")) + created = self.env["odoo.maintainer"] + if missing_maintainer_names: + created = created.sudo().create( + [{"name": name} for name in missing_maintainer_names] + ) + return (maintainers | created).ids + return [] + + @tools.ormcache("name") + def _get_dev_status_id(self, name): + if name: + rec = self.env["odoo.module.dev.status"].search( + [("name", "=", name)], limit=1 + ) + if not rec: + rec = self.env["odoo.module.dev.status"].sudo().create({"name": name}) + return rec.id + return False + + @api.model + def _find(self, branch, module, repo, domain=None): + """Find an `odoo.module.branch` record matching parameters. + + The lookup of the module is in this order: + - search in `repo` + - search among generic modules in other repositories + - search among orphaned modules + + Additional search criteria can be added with `domain`, + e.g. `domain=[('installable', '=', True)]`. + """ + # Look for the module first in the current repository + module_branch = self.browse() + if repo: + module_branch = self._get_module_branch( + branch, module, repo=repo, domain=domain + ) + # Then look among generic modules + if not module_branch: + modules_branch = self._get_module_branch( + branch, + module, + domain=expression.AND( + [ + domain or [], + [("specific", "=", False), ("repository_id", "!=", False)], + ], + ), + ) + module_branch = fields.first(modules_branch) + # Otherwise look for the module among orphaned modules + if not module_branch: + module_branch = self._get_orphaned_module_branch( + branch, module, domain=domain + ) + return module_branch + + @api.model + def _find_or_create(self, branch, module, repo, domain=None): + """Find an `odoo.module.branch` record, or create an orphaned one.""" + module_branch = self._find(branch, module, repo, domain=domain) + # If still not found, create the module as an orphaned module + # (it will hopefully be bound to a repository later) + if not module_branch: + module_branch = self.sudo()._create_orphaned_module_branch(branch, module) + return module_branch + + def _get_dependency_ids(self, repo_branch, depends: list): + dependency_ids = [] + for depend in depends: + module = self._get_module(depend) + dependency = self._find_or_create( + repo_branch.branch_id, module, repo_branch.repository_id + ) + dependency_ids.append(dependency.id) + return dependency_ids + + @tools.ormcache("packages") + def _get_python_dependency_ids(self, packages): + if packages: + dependencies = self.env["odoo.python.dependency"].search( + [("name", "in", packages)] + ) + missing_dependencies = set(packages) - set(dependencies.mapped("name")) + created = self.env["odoo.python.dependency"] + if missing_dependencies: + created = created.sudo().create( + [{"name": package} for package in missing_dependencies] + ) + return (dependencies | created).ids + return [] + + @tools.ormcache("license_name") + def _get_license_id(self, license_name): + if license_name: + license_model = self.env["odoo.license"] + rec = license_model.search([("name", "=", license_name)], limit=1) + if not rec: + rec = license_model.sudo().create({"name": license_name}) + return rec.id + return False + + def _get_module(self, name): + module = self.env["odoo.module"].search([("name", "=", name)]) + if not module: + module = self.env["odoo.module"].sudo().create({"name": name}) + return module + + @api.model + def _get_module_branch_domain(self, branch, module, repo=None, domain=None): + """Return the domain to identify an `odoo.module.branch` record.""" + _domain = [ + ("branch_id", "=", branch.id), + ("module_id", "=", module.id), + ] + if repo: + _domain.append(("repository_id", "=", repo.id)) + elif repo is False: + _domain.append(("repository_id", "=", False)) + if domain: + _domain.extend(domain) + return _domain + + @api.model + def _get_module_branch(self, branch, module, repo=None, domain=None): + """Return the `odoo.module.branch` if it already exists. Do not create it.""" + domain = self._get_module_branch_domain( + branch, module, repo=repo, domain=domain + ) + return self.search(domain) + + @api.model + def _get_orphaned_module_branch_domain(self, branch, module, domain=None): + """Return the domain to identify an orphaned module (without repo).""" + return self._get_module_branch_domain(branch, module, repo=False, domain=domain) + + @api.model + def _get_orphaned_module_branch(self, branch, module, domain=None): + """Return an orphaned module matching `branch` and `module`.""" + domain = self._get_orphaned_module_branch_domain(branch, module, domain=domain) + return self.search(domain) + + @api.model + def _get_unmerged_module_branch_domain(self, branch, module): + """Return the domain to identify an unmerged module (coming from a PR).""" + domain = self._get_module_branch_domain(branch, module) + domain.extend( + [ + ("specific", "=", False), + ("repository_id", "!=", False), + ("pr_url", "!=", False), + ] + ) + return domain + + @api.model + def _get_unmerged_module_branch(self, branch, module): + """Return an unmerged module matching `branch` and `module`.""" + domain = self._get_unmerged_module_branch_domain(branch, module) + return self.search(domain) + + def _create_orphaned_module_branch(self, branch, module): + """Create an orphaned module.""" + values = { + "module_id": module.id, + "branch_id": branch.id, + } + return self.create(values) + + # TODO adds ormcache + def _get_modules_data(self, orgs=None, repositories=None, branches=None): + """Returns modules data matching the criteria. + + E.g.: + + >>> self._get_modules_data( + ... orgs=['OCA'], + ... repositories=['server-env'], + ... branches=['15.0', '16.0'], + ... ) + + """ + domain = self._get_modules_domain(orgs, repositories, branches) + modules = self.search(domain) + data = [] + for module in modules: + data.append(module._to_dict()) + return data + + def _get_modules_domain(self, orgs=None, repositories=None, branches=None): + domain = [ + # Do not return orphans modules + ("org_id", "!=", False), + ("repository_id", "!=", False), + ("branch_id", "!=", False), + ] + if orgs: + domain.append(("org_id", "in", orgs)) + if repositories: + domain.append(("repository_id", "in", repositories)) + if branches: + domain.append(("branch_id", "in", branches)) + return domain + + def _to_dict(self): + """Convert module data to a dictionary.""" + self.ensure_one() + return { + "module": self.module_name, + "branch": self.branch_id.name, + "repository": self.repository_branch_id._to_dict(), + "title": self.title, + "summary": self.summary, + "authors": self.author_ids.mapped("name"), + "maintainers": self.maintainer_ids.mapped("name"), + "depends": self.dependency_ids.mapped("module_name"), + "category": self.category_id.name, + "license": self.license_id.name, + "version": self.version, + "versions": [version._to_dict() for version in self.version_ids], + "development_status": self.development_status_id.name, + "application": self.application, + "installable": self.installable, + "auto_install": self.auto_install, + "external_dependencies": self.external_dependencies, + "is_standard": self.is_standard, + "is_enterprise": self.is_enterprise, + "is_community": self.is_community, + "sloc_python": self.sloc_python, + "sloc_xml": self.sloc_xml, + "sloc_js": self.sloc_js, + "sloc_css": self.sloc_css, + "last_scanned_commit": self.last_scanned_commit, + "addons_path": self.addons_path, + "pr_url": self.pr_url, + } diff --git a/odoo_repository/models/odoo_module_branch_version.py b/odoo_repository/models/odoo_module_branch_version.py new file mode 100644 index 0000000..4e96bc2 --- /dev/null +++ b/odoo_repository/models/odoo_module_branch_version.py @@ -0,0 +1,131 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import api, fields, models + + +class OdooModuleBranchVersion(models.Model): + _name = "odoo.module.branch.version" + _description = "Version of a Odoo Module on a given branch" + _order = "branch_sequence DESC, sequence DESC" + + module_branch_id = fields.Many2one( + comodel_name="odoo.module.branch", + ondelete="cascade", + string="Module", + required=True, + index=True, + ) + branch_id = fields.Many2one( + related="module_branch_id.branch_id", + store=True, + ) + module_id = fields.Many2one( + string="Module ", + related="module_branch_id.module_id", + store=True, + index=True, + ) + module_name = fields.Char( + string="Module Technical Name", + related="module_branch_id.module_name", + store=True, + index=True, + ) + branch_sequence = fields.Integer( + string="Branch Sequence", + related="branch_id.sequence", + store=True, + ) + name = fields.Char(required=True) + manifest_value = fields.Char( + required=True, help="Technical field to host the manifest value." + ) + commit = fields.Char() + has_migration_script = fields.Boolean(default=False) + migration_script_url = fields.Char( + string="Migration Script", + compute="_compute_migration_script_url", + ) + sequence = fields.Integer() + + _sql_constraints = [ + ( + "module_branch_id_name_manifest_value_uniq", + "UNIQUE (module_branch_id, name, manifest_value)", + "This version already exists for this module.", + ), + ] + + @api.model_create_multi + def create(self, vals_list): + res = super().create(vals_list) + res._recompute_sequence() + return res + + def write(self, values): + res = super().write(values) + self._recompute_sequence() + return res + + def _recompute_sequence(self): + """Recompute the 'sequence' field to get versions sorted.""" + self.flush_recordset() + versions_to_recompute = self.search( + [("module_branch_id", "in", self.module_branch_id.ids)] + ) + for version in versions_to_recompute: + query = """ + UPDATE odoo_module_branch_version + SET sequence = ( + SELECT pos.position + FROM ( + SELECT + id, + row_number() OVER ( + ORDER BY string_to_array(name, '.')::int[] + ) AS position + FROM odoo_module_branch_version + WHERE module_branch_id = %(module_branch_id)s + ) as pos + WHERE pos.id = %(id)s + ) + WHERE id = %(id)s; + """ + args = { + "module_branch_id": version.module_branch_id.id, + "id": version.id, + } + self.env.cr.execute(query, args) + self.invalidate_recordset(["sequence"]) + + @api.depends("name", "has_migration_script") + def _compute_migration_script_url(self): + for rec in self: + rec.migration_script_url = False + repo = rec.module_branch_id.repository_id + if rec.has_migration_script: + migration_path = "/".join( + [ + rec.module_branch_id.addons_path or ".", + rec.module_branch_id.module_name, + "migrations", + rec.manifest_value, + ] + ) + rb = rec.module_branch_id.repository_branch_id + branch_name = rb.cloned_branch or rb.branch_id.name + rec.migration_script_url = repo._get_resource_url( + branch_name, migration_path + ) + + def _to_dict(self): + """Convert version data to a dictionary.""" + self.ensure_one() + return { + "name": self.name, + "manifest_value": self.manifest_value, + "commit": self.commit, + "has_migration_script": self.has_migration_script, + "sequence": self.sequence, + } diff --git a/odoo_repository/models/odoo_module_category.py b/odoo_repository/models/odoo_module_category.py new file mode 100644 index 0000000..a957208 --- /dev/null +++ b/odoo_repository/models/odoo_module_category.py @@ -0,0 +1,15 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class OdooModuleCategory(models.Model): + _name = "odoo.module.category" + _description = "Odoo Module Category" + + name = fields.Char(required=True, index=True) + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This module category already exists."), + ] diff --git a/odoo_repository/models/odoo_module_dev_status.py b/odoo_repository/models/odoo_module_dev_status.py new file mode 100644 index 0000000..908a3a1 --- /dev/null +++ b/odoo_repository/models/odoo_module_dev_status.py @@ -0,0 +1,15 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class OdooModuleDevStatus(models.Model): + _name = "odoo.module.dev.status" + _description = "Odoo Module Development Status" + + name = fields.Char(required=True, index=True) + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This development_status already exists."), + ] diff --git a/odoo_repository/models/odoo_python_dependency.py b/odoo_repository/models/odoo_python_dependency.py new file mode 100644 index 0000000..2714025 --- /dev/null +++ b/odoo_repository/models/odoo_python_dependency.py @@ -0,0 +1,16 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class OdooPythonDependency(models.Model): + _name = "odoo.python.dependency" + _description = "Odoo Python Dependency" + _order = "name" + + name = fields.Char(required=True, index=True) + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This Python dependency already exists."), + ] diff --git a/odoo_repository/models/odoo_repository.py b/odoo_repository/models/odoo_repository.py new file mode 100644 index 0000000..f0d0ec3 --- /dev/null +++ b/odoo_repository/models/odoo_repository.py @@ -0,0 +1,680 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import json +import logging +import os +import pathlib +from urllib.parse import urljoin + +import requests + +from odoo import _, api, fields, models, tools +from odoo.exceptions import UserError +from odoo.osv.expression import AND, OR + +from odoo.addons.queue_job.delay import chain +from odoo.addons.queue_job.exception import RetryableJobError +from odoo.addons.queue_job.job import identity_exact + +from ..utils.scanner import RepositoryScannerOdooEnv + +_logger = logging.getLogger(__name__) + + +class OdooRepository(models.Model): + _name = "odoo.repository" + _description = "Odoo Modules Repository" + _order = "sequence, display_name" + + _repositories_path_key = "odoo_repository_storage_path" + + display_name = fields.Char(compute="_compute_display_name", store=True) + active = fields.Boolean(default=True) + sequence = fields.Integer(default=100) + org_id = fields.Many2one( + comodel_name="odoo.repository.org", + ondelete="cascade", + string="Organization", + required=True, + index=True, + ) + name = fields.Char(required=True, index=True) + repo_url = fields.Char( + string="Web URL", + help="Web access to this repository.", + required=True, + ) + to_scan = fields.Boolean( + default=True, + help="Scan this repository to collect data.", + ) + clone_url = fields.Char( + string="Clone URL", + help="Used to clone the repository.", + ) + clone_name = fields.Char( + help=( + "Force the name of the cloned repository (folder on disk). " + "If not set, the name of the repository will be used." + ), + ) + repo_type = fields.Selection( + selection=[ + ("github", "GitHub"), + ("gitlab", "GitLab"), + ], + ) + ssh_key_id = fields.Many2one( + comodel_name="ssh.key", + ondelete="restrict", + string="SSH Key", + help="SSH key used to clone/fetch this repository.", + ) + token_id = fields.Many2one( + comodel_name="authentication.token", + ondelete="restrict", + string="Token", + help="Token used to clone/fetch this repository.", + ) + active = fields.Boolean(default=True) + addons_path_ids = fields.Many2many( + comodel_name="odoo.repository.addons_path", + string="Addons Path", + help="Relative path of folders in this repository hosting Odoo modules", + ) + branch_ids = fields.One2many( + comodel_name="odoo.repository.branch", + inverse_name="repository_id", + string="Branches", + ) + scan_weekday_ids = fields.Many2many( + comodel_name="time.weekday", + string="Scanning days", + help=( + "Limit scanning of this repository by the scheduled action to " + "certain days only. If not defined, the scan will happen every day." + ), + ) + manual_branches = fields.Boolean( + string="Configure branches manually", + help=( + "By default repository branches follows the configured Odoo versions " + "(e.g: 17.0, 18.0...). Enable this option to configure your own branches." + ), + ) + specific = fields.Boolean( + help=( + "Host specific modules (that are not generic). " + "Used for project repositories." + ), + ) + module_ids = fields.One2many( + comodel_name="odoo.module.branch", + inverse_name="repository_id", + string="Modules", + readonly=True, + ) + + @api.model + def default_get(self, fields_list): + """'default_get' method overloaded.""" + res = super().default_get(fields_list) + if "addons_path_ids" not in res: + res["addons_path_ids"] = [ + ( + 4, + self.env.ref( + "odoo_repository.odoo_repository_addons_path_community" + ).id, + ) + ] + return res + + @api.depends("org_id", "name") + def _compute_github_url(self): + for rec in self: + rec.github_url = f"{rec.org_id.github_url}/{rec.name}" + + _sql_constraints = [ + ( + "org_id_name_uniq", + "UNIQUE (org_id, name)", + "This repository already exists.", + ), + ] + + @api.depends("org_id.name", "name") + def _compute_display_name(self): + for rec in self: + rec.display_name = f"{rec.org_id.name}/{rec.name}" + + @api.onchange("repo_url", "to_scan", "clone_url") + def _onchange_repo_url(self): + if not self.repo_url: + return + for type_, __ in self._fields["repo_type"].selection: + if type_ not in self.repo_url: + continue + self.repo_type = type_ + if not self.clone_url and self.to_scan: + self.clone_url = self.repo_url + break + + def _get_odoo_branches_to_scan(self): + self.ensure_one() + if self.manual_branches: + return self.branch_ids.branch_id + return self.env["odoo.branch"]._get_all_odoo_versions(active_test=True) + + def _cron_scanner_domain(self): + today = fields.Date.today() + weekday = today.weekday() + return AND( + [ + [("to_scan", "=", True)], + OR( + [ + [("scan_weekday_ids.name", "=", weekday)], + [("scan_weekday_ids", "=", False)], + ] + ), + ] + ) + + @api.model + def cron_scanner(self, branches=None, force=False): + """Scan and collect Odoo repositories data. + + As the scanner is run on the same server than Odoo, a special class + `RepositoryScannerOdooEnv` is used so the scanner can request Odoo + through an environment (api.Environment). + + `branches` parameter allows to filter the `odoo.branch` to take into + account for the scan, e.g. `branches=["16.0", "18.0"]`. + """ + repositories = self.search(self._cron_scanner_domain()) + branches_ = self.env["odoo.branch"]._get_all_odoo_versions(active_test=True) + if branches: + branches_ = branches_.filtered(lambda br: br.name in branches) + for repo in repositories: + repo.action_scan(branch_ids=branches_.ids, force=force, raise_exc=False) + + def _check_config(self): + # Check the configuration of repositories folder + key = self._repositories_path_key + repositories_path = self.env["ir.config_parameter"].sudo().get_param(key, "") + if not repositories_path: + raise UserError( + _( + f"Please define the '{key}' system parameter to " + "clone repositories in the folder of your choice." + ) + ) + # Ensure the folder exists + pathlib.Path(repositories_path).mkdir(parents=True, exist_ok=True) + + def _check_existing_jobs(self, raise_exc=True): + """Check if a scan is already triggered for this repository.""" + self.ensure_one() + existing_job = ( + self.env["queue.job"] + .sudo() + .search( + [ + ("model_name", "=", self._name), + ("records", "ilike", f'%"ids": [{self.id}]%'), + ( + "state", + "in", + [ + "wait_dependencies", + "pending", + "enqueued", + "started", + ], + ), + ], + limit=1, + ) + ) + if existing_job: + msg = _("A scan is already ongoing for repository %s") % self.display_name + if raise_exc: + raise UserError(msg) + _logger.warning(msg) + return True + return False + + def action_scan(self, branch_ids=None, force=False, raise_exc=True): + """Scan the whole repository.""" + self._check_config() + for rec in self: + if not rec.to_scan: + continue + if rec._check_existing_jobs(raise_exc=raise_exc): + continue + # Get branch records to scan + branches = rec._get_odoo_branches_to_scan() + if branch_ids: + branches = branches & self.env["odoo.branch"].search( + [("id", "in", branch_ids)] + ) + if not branches: + continue + # Create a list of tuples ({odoo_version}, {branch_name}) + versions_branches = [(branch.name, branch.name) for branch in branches] + if rec.manual_branches: + versions_branches = [ + (rb.branch_id.name, rb.cloned_branch or rb.branch_id.name) + for rb in rec.branch_ids + if rb.branch_id in branches + ] + if force: + rec._reset_scanned_commits(branch_ids=branch_ids) + # Scan repository branches sequentially as they need to be checked out + # to perform the analysis + # Here the launched job is responsible to: + # 1) detect modules updated on the first branch + # 2) spawn jobs to scan each module on that branch + # 3) spawn a job to update the last scanned commit of the repo/branch + # 4) spawn the next job responsible to detect modules updated + # on the next branch + version_branch = versions_branches[0] + next_versions_branches = versions_branches[1:] + job = rec._create_job_detect_modules_to_scan_on_branch( + version_branch, next_versions_branches, versions_branches + ) + job.delay() + return True + + def _create_job_detect_modules_to_scan_on_branch( + self, version_branch, next_versions_branches, all_versions_branches + ): + self.ensure_one() + version, branch = version_branch + branch_str = branch + if version != branch: + branch_str = f"{branch} ({version})" + delayable = self.delayable( + description=f"Detect modules to scan in {self.display_name}#{branch_str}", + identity_key=identity_exact, + ) + return delayable._detect_modules_to_scan_on_branch( + version_branch, next_versions_branches, all_versions_branches + ) + + def _detect_modules_to_scan_on_branch( + self, version_branch, next_versions_branches, all_versions_branches + ): + """Detect the modules to scan on `branch`. + + It will spawn a job for each module to scan, and two other jobs to: + - update the last scanned commit on the repo/branch + - scan the next branch (so each branch is scanned in cascade) + + This ensure to scan different branches sequentially for a given repository. + """ + version, branch = version_branch + try: + # Get the list of modules updated since last scan + params = self._prepare_scanner_parameters(version, branch) + scanner = RepositoryScannerOdooEnv(**params) + data = scanner.detect_modules_to_scan() + # Prepare all subsequent jobs based on modules to scan + jobs = self._create_subsequent_jobs( + version_branch, next_versions_branches, all_versions_branches, data + ) + # Chain them altogether + if jobs: + chain(*jobs).delay() + except Exception as exc: + raise RetryableJobError("Scanner error") from exc + + def _create_subsequent_jobs( + self, version_branch, next_versions_branches, all_versions_branches, data + ): + jobs = [] + version, branch = version_branch + # Spawn one job per module to scan + for data_ in data.get("addons_paths", {}).values(): + for module_path in data_["modules_to_scan"]: + job = self._create_job_scan_module_on_branch( + version, branch, module_path, data_["specs"] + ) + jobs.append(job) + # + another one to update the last scanned commit of the repository + if data.get("repo_branch_id"): + job = self._create_job_update_last_scanned_commit( + data["repo_branch_id"], + data["last_fetched_commit"], + ) + jobs.append(job) + # + another one to detect modules to scan on the next branch + version_branch = next_versions_branches and next_versions_branches[0] + next_versions_branches = next_versions_branches[1:] + if version_branch: + jobs.append( + self._create_job_detect_modules_to_scan_on_branch( + version_branch, next_versions_branches, all_versions_branches + ) + ) + return jobs + + def _create_job_scan_module_on_branch(self, version, branch, module_path, specs): + self.ensure_one() + branch_str = branch + if version != branch: + branch_str = f"{branch} ({version})" + delayable = self.delayable( + description=f"Scan {self.display_name}#{branch_str} - {module_path}", + identity_key=identity_exact, + ) + return delayable._scan_module_on_branch(version, branch, module_path, specs) + + def _scan_module_on_branch(self, version, branch, module_path, specs): + """Scan `module_path` from `branch`.""" + try: + params = self._prepare_scanner_parameters(version, branch) + scanner = RepositoryScannerOdooEnv(**params) + return scanner.scan_module(module_path, specs) + except Exception as exc: + raise RetryableJobError("Scanner error") from exc + + def _create_job_update_last_scanned_commit( + self, repo_branch_id, last_scanned_commit, last_scan=False + ): + self.ensure_one() + repo_branch_model = self.env["odoo.repository.branch"] + repo_branch = repo_branch_model.browse(repo_branch_id).exists() + delayable = repo_branch.delayable( + description=f"Update last scanned commit of {repo_branch.display_name}", + identity_key=identity_exact, + ) + return delayable._update_last_scanned_commit(last_scanned_commit) + + def _reset_scanned_commits(self, branch_ids=None): + """Reset the scanned commits. + + This will make the next repository scan restarting from the beginning, + and thus making it slower. + """ + self.ensure_one() + if branch_ids is None: + branch_ids = self.branch_ids.branch_id.ids + repo_branches = self.branch_ids.filtered( + lambda rb: rb.branch_id.id in branch_ids + ) + repo_branches.write({"last_scanned_commit": False}) + repo_branches.module_ids.sudo().write({"last_scanned_commit": False}) + + def _get_token(self): + """Return the first available token found for this repository. + + It will check the available tokens in this order: + - specific token linked to this repository + - default token defined in the global settings + - token defined through an environment variable + """ + self.ensure_one() + return ( + self.token_id.token + or self.env.company.config_odoo_repository_default_token_id.token + or os.environ.get("GITHUB_TOKEN") + ) + + def _prepare_scanner_parameters(self, version, branch): + ir_config = self.env["ir.config_parameter"] + repositories_path = ir_config.sudo().get_param(self._repositories_path_key) + return { + "org": self.org_id.name, + "name": self.name, + "clone_url": self.clone_url, + "version": version, + "branch": branch, + "addons_paths_data": self.addons_path_ids.read( + [ + "relative_path", + "is_standard", + "is_enterprise", + "is_community", + ] + ), + "repositories_path": repositories_path, + "repo_type": self.repo_type, + "ssh_key": self.ssh_key_id.private_key, + "token": self._get_token(), + "workaround_fs_errors": ( + self.env.company.config_odoo_repository_workaround_fs_errors + ), + "clone_name": self.clone_name, + "env": self.env, + } + + def action_force_scan(self, branch_ids=None, raise_exc=True): + """Force the scan of the repositories. + + It will restart the scan without considering the last scanned commit, + overriding already collected module data if any. + """ + self.ensure_one() + return self.action_scan(branch_ids=branch_ids, force=True, raise_exc=raise_exc) + + @api.model + def cron_fetch_data(self, branches=None, force=False): + """Fetch Odoo repositories data from the main node (if any).""" + main_node_url = ( + self.env["ir.config_parameter"] + .sudo() + .get_param("odoo_repository_main_node_url") + ) + if not main_node_url: + return False + branch_domain = [] + if branches: + branch_domain.append(("name", "in", branches)) + branches = self.env["odoo.branch"].search(branch_domain) + branch_names = ",".join(branches.mapped("name")) + url = f"{main_node_url}?branches=%s" % branch_names + try: + response = requests.get(url, timeout=60) + except Exception as exc: + raise UserError(_("Unable to fetch data from %s") % main_node_url) from exc + else: + if response.status_code == 200: + try: + data = json.loads(response.text) + except json.decoder.JSONDecodeError as exc: + raise UserError( + _("Unable to decode data received from %s") % main_node_url + ) from exc + else: + self._import_data(data) + + def _import_data(self, data): + for module_data in data: + # TODO Move these methods to 'odoo.module.branch'? + values = self._prepare_module_branch_values(module_data) + self._create_or_update_module_branch(values, module_data) + + def _prepare_module_branch_values(self, data): + # Get branch, repository and technical module + branch = self.env["odoo.branch"].search([("name", "=", data["branch"])]) + org = self._get_repository_org(data["repository"]["org"]) + repository = self._get_repository( + org.id, data["repository"]["name"], data["repository"] + ) + repository_branch = self._get_repository_branch( + org.id, repository.id, branch.id, data["repository"] + ) + + mb_model = self.env["odoo.module.branch"] + module = mb_model._get_module(data["module"]) + # Prepare values + category_id = mb_model._get_module_category_id(data["category"]) + author_ids = mb_model._get_author_ids(tuple(data["authors"])) + maintainer_ids = mb_model._get_maintainer_ids(tuple(data["maintainers"])) + dev_status_id = mb_model._get_dev_status_id(data["development_status"]) + dependency_ids = mb_model._get_dependency_ids( + repository_branch, data["depends"] + ) + external_dependencies = data["external_dependencies"] + python_dependency_ids = mb_model._get_python_dependency_ids( + tuple(external_dependencies.get("python", [])) + ) + license_id = mb_model._get_license_id(data["license"]) + versions_values = self._prepare_version_ids_values( + repository_branch, module, data["versions"] + ) + values = { + "repository_branch_id": repository_branch.id, + "branch_id": repository_branch.branch_id.id, + "module_id": module.id, + "title": data["title"], + "summary": data["summary"], + "category_id": category_id, + "author_ids": [(6, 0, author_ids)], + "maintainer_ids": [(6, 0, maintainer_ids)], + "dependency_ids": [(6, 0, dependency_ids)], + "external_dependencies": external_dependencies, + "python_dependency_ids": [(6, 0, python_dependency_ids)], + "license_id": license_id, + "version": data["version"], + "version_ids": versions_values, + "development_status_id": dev_status_id, + "installable": data["installable"], + "auto_install": data["auto_install"], + "application": data["application"], + "is_standard": data["is_standard"], + "is_enterprise": data["is_enterprise"], + "is_community": data["is_community"], + "sloc_python": data["sloc_python"], + "sloc_xml": data["sloc_xml"], + "sloc_js": data["sloc_js"], + "sloc_css": data["sloc_css"], + "last_scanned_commit": data["last_scanned_commit"], + "pr_url": data["pr_url"], + } + return values + + def _prepare_version_ids_values(self, repo_branch, module, versions: list[dict]): + version_ids = [] + for version in versions: + version_model = self.env["odoo.module.branch.version"] + rec = version_model.search( + [ + ("module_branch_id.branch_id", "=", repo_branch.branch_id.id), + ("module_branch_id.module_id", "=", module.id), + ("name", "=", version["name"]), + ], + limit=1, + ) + if rec: + version_ids.append(fields.Command.update(rec.id, version)) + else: + version_ids.append(fields.Command.create(version)) + return version_ids + + def _create_or_update_module_branch(self, values, raw_data): + mb_model = self.env["odoo.module.branch"] + rec = mb_model.search( + [ + ("module_id", "=", values["module_id"]), + # Module could have been already created to satisfy dependencies + # (without 'repository_branch_id' set) + "|", + ("repository_branch_id", "=", values["repository_branch_id"]), + ("branch_id", "=", values["branch_id"]), + ], + limit=1, + ) + values = self._pre_create_or_update_module_branch(rec, values, raw_data) + if rec: + rec.sudo().write(values) + else: + rec = mb_model.sudo().create(values) + self._post_create_or_update_module_branch(rec, values, raw_data) + return rec + + def _pre_create_or_update_module_branch(self, rec, values, raw_data): + """Hook executed before the creation or update of `rec`. Return values.""" + return values + + def _post_create_or_update_module_branch(self, rec, values, raw_data): + """Hook executed after the creation or update of `rec`.""" + + @tools.ormcache("name") + def _get_repository_org(self, name): + rec = self.env["odoo.repository.org"].search([("name", "=", name)], limit=1) + if not rec: + rec = self.env["odoo.repository.org"].sudo().create({"name": name}) + return rec + + @tools.ormcache("org_id", "name") + def _get_repository(self, org_id, name, data): + rec = self.env["odoo.repository"].search( + [ + ("org_id", "=", org_id), + ("name", "=", name), + ], + limit=1, + ) + values = { + "org_id": org_id, + "name": name, + "repo_url": data["repo_url"], + "repo_type": data["repo_type"], + "active": data["active"], + } + if rec: + rec.sudo().write(values) + else: + rec = self.env["odoo.repository"].sudo().create(values) + return rec + + @tools.ormcache("org_id", "repository_id", "branch_id") + def _get_repository_branch(self, org_id, repository_id, branch_id, data): + rec = self.env["odoo.repository.branch"].search( + [ + ("repository_id", "=", repository_id), + ("branch_id", "=", branch_id), + ], + limit=1, + ) + values = { + "repository_id": repository_id, + "branch_id": branch_id, + "last_scanned_commit": data["last_scanned_commit"], + } + if rec: + rec.sudo().write(values) + else: + rec = self.env["odoo.repository.branch"].sudo().create(values) + return rec + + def _get_resource_url(self, branch, path): + self.ensure_one() + # NOTE: GitHub and GitLab supports the same URL pattern + url = "/".join(["tree", branch, path]) + return urljoin(self.repo_url + "/", url) + + def unlink(self): + # There is no deletion on cascade policy by default, but for specific + # repositories we want to remove specific modules anyway. + # This will also avoid to raise UNIQUE constraint + # 'odoo_module_branch_uniq_null(module_id, branch_id)' if module names + # are shared between repositories. + for rec in self: + if rec.specific: + rec.branch_ids.module_ids.sudo().unlink() + return super().unlink() + + def open_modules(self): + self.ensure_one() + xml_id = "odoo_repository.odoo_module_branch_action" + action = self.env["ir.actions.actions"]._for_xml_id(xml_id) + action["domain"] = [("repository_id", "=", self.id)] + action["context"] = {"search_default_installable": True} + if len(self.module_ids.branch_id) > 1: + action["context"]["search_default_group_by_branch_id"] = True + return action diff --git a/odoo_repository/models/odoo_repository_addons_path.py b/odoo_repository/models/odoo_repository_addons_path.py new file mode 100644 index 0000000..7c8d192 --- /dev/null +++ b/odoo_repository/models/odoo_repository_addons_path.py @@ -0,0 +1,40 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class OdooRepositoryAddonsPath(models.Model): + _name = "odoo.repository.addons_path" + _description = "Addons" + + relative_path = fields.Char(required=True) + is_standard = fields.Boolean( + string="Standard?", + help="Does this folder contain modules from Odoo S.A.?", + default=False, + ) + is_enterprise = fields.Boolean( + string="Enterprise?", + help=( + "Does this folder contain Enterprise modules?\n" + "(from Odoo S.A., a contributor or your organization)" + ), + default=False, + ) + is_community = fields.Boolean( + string="Community Contribution?", + help=( + "Does this folder contain Odoo generic community modules?\n" + "(from OCA, a contributor or your organization)" + ), + default=False, + ) + + _sql_constraints = [ + ( + "addons_path_uniq", + "UNIQUE (relative_path, is_standard, is_enterprise, is_community)", + "This addons-path already exists.", + ), + ] diff --git a/odoo_repository/models/odoo_repository_branch.py b/odoo_repository/models/odoo_repository_branch.py new file mode 100644 index 0000000..d748c2a --- /dev/null +++ b/odoo_repository/models/odoo_repository_branch.py @@ -0,0 +1,98 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import api, fields, models + + +class OdooRepositoryBranch(models.Model): + _name = "odoo.repository.branch" + _description = "Odoo Modules Repository Branch" + + name = fields.Char(compute="_compute_name", store=True, index=True) + repository_id = fields.Many2one( + comodel_name="odoo.repository", + ondelete="cascade", + string="Repository", + required=True, + index=True, + readonly=True, + ) + manual_branches = fields.Boolean( + related="repository_id.manual_branches", + store=True, + ) + specific = fields.Boolean( + related="repository_id.specific", + store=True, + ) + branch_id = fields.Many2one( + comodel_name="odoo.branch", + ondelete="cascade", + string="Odoo Version", + required=True, + index=True, + ) + cloned_branch = fields.Char( + help=( + "Force the branch to clone (optional). Used on repositories with " + "'Configure branches manually' option enabled." + ), + ) + module_ids = fields.One2many( + comodel_name="odoo.module.branch", + inverse_name="repository_branch_id", + string="Modules", + readonly=True, + ) + last_scanned_commit = fields.Char(readonly=True) + active = fields.Boolean(compute="_compute_active", store=True) + + _sql_constraints = [ + ( + "repository_id_branch_id_uniq", + "UNIQUE (repository_id, branch_id)", + "This branch already exists for this repository.", + ), + ] + + @api.depends("repository_id.display_name", "branch_id.name") + def _compute_name(self): + for rec in self: + rec.name = f"{rec.repository_id.display_name}#{rec.branch_id.name}" + + @api.depends("repository_id.active", "branch_id.active") + def _compute_active(self): + for rec in self: + rec.active = all((rec.repository_id.active, rec.branch_id.active)) + + def action_scan(self, force=False, raise_exc=True): + """Scan the repository/branch.""" + return self.repository_id.action_scan( + branch_ids=self.branch_id.ids, force=force, raise_exc=raise_exc + ) + + def action_force_scan(self, raise_exc=True): + """Force the scan of the repository/branch. + + It will restart the scan without considering the last scanned commit, + overriding already collected module data if any. + """ + return self.action_scan(force=True, raise_exc=raise_exc) + + def _to_dict(self): + """Convert branch repository data to a dictionary.""" + self.ensure_one() + return { + "org": self.repository_id.org_id.name, + "name": self.repository_id.name, + "repo_url": self.repository_id.repo_url, + "repo_type": self.repository_id.repo_type, + "active": self.repository_id.active, + "branch": self.branch_id.name, + "last_scanned_commit": self.last_scanned_commit, + } + + def _update_last_scanned_commit(self, last_scanned_commit): + """Update the last scanned commit. Called by job.""" + self.ensure_one() + self.last_scanned_commit = last_scanned_commit diff --git a/odoo_repository/models/odoo_repository_org.py b/odoo_repository/models/odoo_repository_org.py new file mode 100644 index 0000000..38605aa --- /dev/null +++ b/odoo_repository/models/odoo_repository_org.py @@ -0,0 +1,23 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import api, fields, models + +from ..utils.github import GITHUB_URL + + +class OdooRepositoryOrg(models.Model): + _name = "odoo.repository.org" + _description = "Odoo Repository Organization" + + name = fields.Char(required=True, index=True) + github_url = fields.Char(string="GitHub URL", compute="_compute_github_url") + + @api.depends("name") + def _compute_github_url(self): + for rec in self: + rec.github_url = f"{GITHUB_URL}/{rec.name}" + + _sql_constraints = [ + ("name_uniq", "UNIQUE (name)", "This organization already exists."), + ] diff --git a/odoo_repository/models/res_company.py b/odoo_repository/models/res_company.py new file mode 100644 index 0000000..355c894 --- /dev/null +++ b/odoo_repository/models/res_company.py @@ -0,0 +1,26 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class ResCompany(models.Model): + _inherit = "res.company" + + config_odoo_repository_default_token_id = fields.Many2one( + comodel_name="authentication.token", + string="Default Authentication Token", + help=( + "Default token used to clone repositories and authenticate " + "on API like GitHub." + ), + ) + config_odoo_repository_workaround_fs_errors = fields.Boolean( + string="Workaround FS errors", + help=( + "Fix file system permissions when cloning repositories. " + "Errors could be triggered on some file systems when git tries to " + "execute 'chown' commands on its internal configuration files. " + "This option will workaround this issue." + ), + ) diff --git a/odoo_repository/models/res_config_settings.py b/odoo_repository/models/res_config_settings.py new file mode 100644 index 0000000..ef16923 --- /dev/null +++ b/odoo_repository/models/res_config_settings.py @@ -0,0 +1,23 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class ResConfigSettings(models.TransientModel): + _inherit = "res.config.settings" + + config_odoo_repository_storage_path = fields.Char( + string="Storage local path", config_parameter="odoo_repository_storage_path" + ) + config_odoo_repository_workaround_fs_errors = fields.Boolean( + related="company_id.config_odoo_repository_workaround_fs_errors", + readonly=False, + ) + config_odoo_repository_default_token_id = fields.Many2one( + related="company_id.config_odoo_repository_default_token_id", + readonly=False, + ) + config_odoo_repository_main_node_url = fields.Char( + string="Main Node", config_parameter="odoo_repository_main_node_url" + ) diff --git a/odoo_repository/models/ssh_key.py b/odoo_repository/models/ssh_key.py new file mode 100644 index 0000000..1061133 --- /dev/null +++ b/odoo_repository/models/ssh_key.py @@ -0,0 +1,12 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields, models + + +class SSHKey(models.Model): + _name = "ssh.key" + _description = "SSH private key" + + name = fields.Char(required=True) + private_key = fields.Text(required=True, help="SSH private key without passphrase.") diff --git a/odoo_repository/pyproject.toml b/odoo_repository/pyproject.toml new file mode 100644 index 0000000..4231d0c --- /dev/null +++ b/odoo_repository/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/odoo_repository/readme/CONTRIBUTORS.md b/odoo_repository/readme/CONTRIBUTORS.md new file mode 100644 index 0000000..1775292 --- /dev/null +++ b/odoo_repository/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- Camptocamp + - Sébastien Alix \ diff --git a/odoo_repository/readme/DESCRIPTION.md b/odoo_repository/readme/DESCRIPTION.md new file mode 100644 index 0000000..2fea33b --- /dev/null +++ b/odoo_repository/readme/DESCRIPTION.md @@ -0,0 +1,6 @@ +Base module to host data collected from Odoo repositories. + +It allows you to: +- declare the Odoo versions (last 3 versions by default) +- declare repositories containing modules (Odoo and OCA repositories included by default) +- scan these repositories to collect modules informations per Odoo version diff --git a/odoo_repository/security/ir.model.access.csv b/odoo_repository/security/ir.model.access.csv new file mode 100644 index 0000000..db75b49 --- /dev/null +++ b/odoo_repository/security/ir.model.access.csv @@ -0,0 +1,27 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_authentication_token_user,authentication_token_user,model_authentication_token,group_odoo_repository_user,0,0,0,0 +access_authentication_manager,authentication_token_manager,model_authentication_token,group_odoo_repository_manager,1,1,1,1 +access_ssh_key_user,ssh_key_user,model_ssh_key,group_odoo_repository_user,0,0,0,0 +access_ssh_key_manager,ssh_key_manager,model_ssh_key,group_odoo_repository_manager,1,1,1,1 +access_odoo_author_user,odoo_author_user,model_odoo_author,group_odoo_repository_user,1,0,0,0 +access_odoo_branch_user,odoo_branch_user,model_odoo_branch,group_odoo_repository_user,1,0,0,0 +access_odoo_branch_manager,odoo_branch_manager,model_odoo_branch,group_odoo_repository_manager,1,1,1,1 +access_odoo_license_user,odoo_license_user,model_odoo_license,group_odoo_repository_user,1,0,0,0 +access_odoo_maintainer_user,odoo_maintainer_user,model_odoo_maintainer,group_odoo_repository_user,1,0,0,0 +access_odoo_module_user,odoo_module_user,model_odoo_module,group_odoo_repository_user,1,0,0,0 +access_odoo_module_manager,odoo_module_manager,model_odoo_module,group_odoo_repository_manager,1,0,1,1 +access_odoo_module_category_user,odoo_module_category_user,model_odoo_module_category,group_odoo_repository_user,1,0,0,0 +access_odoo_module_dev_status_user,odoo_module_dev_status_user,model_odoo_module_dev_status,group_odoo_repository_user,1,0,0,0 +access_odoo_python_dependency_user,odoo_python_dependency_user,model_odoo_python_dependency,group_odoo_repository_user,1,0,0,0 +access_odoo_repository_addons_path_user,odoo_repository_addons_path_user,model_odoo_repository_addons_path,group_odoo_repository_user,1,0,0,0 +access_odoo_repository_addons_path_manager,odoo_repository_addons_path_manager,model_odoo_repository_addons_path,group_odoo_repository_manager,1,1,1,1 +access_odoo_repository_org_user,odoo_repository_org_user,model_odoo_repository_org,group_odoo_repository_user,1,0,0,0 +access_odoo_repository_org_manager,odoo_repository_org_manager,model_odoo_repository_org,group_odoo_repository_manager,1,1,1,1 +access_odoo_repository_user,odoo_repository_user,model_odoo_repository,group_odoo_repository_user,1,0,0,0 +access_odoo_repository_manager,odoo_repository_manager,model_odoo_repository,group_odoo_repository_manager,1,1,1,1 +access_odoo_repository_branch_user,odoo_repository_branch_user,model_odoo_repository_branch,group_odoo_repository_user,1,0,0,0 +access_odoo_repository_branch_manager,odoo_repository_branch_manager,model_odoo_repository_branch,group_odoo_repository_manager,1,1,1,1 +access_odoo_module_branch_user,odoo_module_branch_user,model_odoo_module_branch,group_odoo_repository_user,1,0,0,0 +access_odoo_module_branch_manager,odoo_module_branch_manager,model_odoo_module_branch,group_odoo_repository_manager,1,1,0,0 +access_odoo_module_branch_version_user,odoo_module_branch_version_user,model_odoo_module_branch_version,group_odoo_repository_user,1,0,0,0 +access_odoo_module_branch_version_manager,odoo_module_branch_version_manager,model_odoo_module_branch_version,group_odoo_repository_manager,1,1,1,1 diff --git a/odoo_repository/security/res_groups.xml b/odoo_repository/security/res_groups.xml new file mode 100644 index 0000000..ca6fa70 --- /dev/null +++ b/odoo_repository/security/res_groups.xml @@ -0,0 +1,23 @@ + + + + + Odoo MCA + 50 + + + + Odoo MCA User + + + + + Odoo MCA Manager + + + + diff --git a/odoo_repository/static/description/README b/odoo_repository/static/description/README new file mode 100644 index 0000000..1183e26 --- /dev/null +++ b/odoo_repository/static/description/README @@ -0,0 +1,4 @@ +Icon generated with https://spilymp.github.io/ibo/ + Set: Font Awesome 5 Solid & Regular + Class: fa-shapes + Background color: 255 / 104 / 10 (RGB) diff --git a/odoo_repository/static/description/icon.png b/odoo_repository/static/description/icon.png new file mode 100644 index 0000000..1a09a5f Binary files /dev/null and b/odoo_repository/static/description/icon.png differ diff --git a/odoo_repository/static/description/index.html b/odoo_repository/static/description/index.html new file mode 100644 index 0000000..9daa823 --- /dev/null +++ b/odoo_repository/static/description/index.html @@ -0,0 +1,440 @@ + + + + + +README.rst + + + +
+ + + +Odoo Community Association + +
+

Odoo Repositories Data

+ +

Beta License: AGPL-3 OCA/module-composition-analysis Translate me on Weblate Try me on Runboat

+

Base module to host data collected from Odoo repositories.

+

It allows you to:

+
    +
  • declare the Odoo versions (last 3 versions by default)
    • +
  • declare repositories containing modules (Odoo and OCA repositories +included by default)
    • +
  • scan these repositories to collect modules informations per Odoo +version
    • +
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • Camptocamp
    • +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/module-composition-analysis project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+
+ + diff --git a/odoo_repository/tests/__init__.py b/odoo_repository/tests/__init__.py new file mode 100644 index 0000000..a806756 --- /dev/null +++ b/odoo_repository/tests/__init__.py @@ -0,0 +1,6 @@ +from . import test_utils +from . import test_base_scanner +from . import test_repository_scanner +from . import test_odoo_repository_scan +from . import test_sync_node +from . import test_odoo_module_branch diff --git a/odoo_repository/tests/common.py b/odoo_repository/tests/common.py new file mode 100644 index 0000000..9dfe8d2 --- /dev/null +++ b/odoo_repository/tests/common.py @@ -0,0 +1,193 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import logging +import os +import pathlib +import re +import tempfile +from unittest.mock import patch + +import git +import psutil +from oca_port.tests.common import CommonCase + +from odoo.tests.common import TransactionCase + +_logger = logging.getLogger(__name__) + + +class Common(TransactionCase, CommonCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + CommonCase.setUpClass() + cls.env = cls.env(context=dict(cls.env.context, tracking_disable=True)) + cls.repositories_path = tempfile.mkdtemp() + cls.env["ir.config_parameter"].set_param( + "odoo_repository_storage_path", cls.repositories_path + ) + cls._apply_git_config() + cls._handle_cleanup() + + def setUp(self): + super().setUp() + # Leverage the existing test class from 'oca_port' to bootstrap + # temporary git repositories to run tests + CommonCase.setUp(self) + self.repo_name = pathlib.Path(self.repo_upstream_path).parts[-1] + self.org = self.env["odoo.repository.org"].create({"name": self.fork_org}) + self.odoo_repository = self.env["odoo.repository"].create( + { + "org_id": self.org.id, + "name": self.repo_name, + "repo_url": self.repo_upstream_path, + "clone_url": self.repo_upstream_path, + "repo_type": "github", + } + ) + # branch1 + self.branch1_name = self.source1.split("/")[1] + self.branch = ( + self.env["odoo.branch"] + .with_context(active_test=False) + .search([("name", "=", self.branch1_name)]) + ) + if not self.branch: + self.branch = self.env["odoo.branch"].create( + { + "name": self.branch1_name, + } + ) + self.branch.active = True + # branch2 + self.branch2_name = self.source2.split("/")[1] + self.branch2 = ( + self.env["odoo.branch"] + .with_context(active_test=False) + .search([("name", "=", self.branch2_name)]) + ) + if not self.branch2: + self.branch2 = self.env["odoo.branch"].create( + { + "name": self.branch2_name, + } + ) + self.branch2.active = True + # branch3 + self.branch3_name = self.target2.split("/")[1] + # technical module + self.module_name = self.addon + self.module_branch_model = self.env["odoo.module.branch"] + + @classmethod + def _apply_git_config(cls): + """Configure git (~/.gitconfig) if no config file exists.""" + git_cfg = pathlib.Path(os.path.expanduser("~/.gitconfig")) + if git_cfg.exists(): + return + os.system("git config --global user.email 'test@example.com'") + os.system("git config --global user.name 'test'") + + def _patch_github_class(self): + res = super()._patch_github_class() + # Patch helper method part of 'odoo_repository' module as well + self.patcher2 = patch("odoo.addons.odoo_repository.utils.github.request") + github_request = self.patcher2.start() + github_request.return_value = {} + self.addCleanup(self.patcher2.stop) + return res + + def _update_module_version_on_branch(self, branch, version): + """Change module version on a given branch, and commit the change.""" + repo = git.Repo(self.repo_upstream_path) + repo.git.checkout(branch) + # Update version in manifest file + lines = [] + with open(self.manifest_path, "r+") as manifest: + for line in manifest: + pattern = r".*version['\"]:\s['\"]([\d.]+).*" + match = re.search(pattern, line) + if match: + current_version = match.group(1) + line = line.replace(current_version, version) + lines.append(line) + with open(self.manifest_path, "r+") as manifest: + manifest.writelines(lines) + # Commit + repo.index.add(self.manifest_path) + commit = repo.index.commit(f"[IMP] {self.addon}: bump version to {version}") + del repo + return commit.hexsha + + def _update_module_installable_on_branch(self, branch, installable=True): + repo = git.Repo(self.repo_upstream_path) + repo.git.checkout(branch) + # Update installable key in manifest file + lines = [] + with open(self.manifest_path, "r+") as manifest: + for line in manifest: + pattern = r".*installable[`\"]:\s(\b[A-Z,a-z]+),.*" + match = re.search(pattern, line) + if match: + current_value = match.group(1) + line = line.replace(current_value, str(installable)) + lines.append(line) + with open(self.manifest_path, "r+") as manifest: + manifest.writelines(lines) + # Commit + repo.index.add(self.manifest_path) + commit = repo.index.commit( + f"[IMP] {self.addon}: make installable={installable}" + ) + del repo + return commit.hexsha + + def _run_odoo_repository_action_scan(self, branch_id, force=False): + """Run `action_scan` for given `branch_id` on the Odoo repository.""" + self.odoo_repository.with_context(queue_job__no_delay=True).action_scan( + branch_ids=[branch_id], force=force + ) + + def _create_odoo_module(self, name): + return self.env["odoo.module"].create({"name": name}) + + def _create_odoo_repository_branch(self, repo, branch, **values): + vals = { + "repository_id": repo.id, + "branch_id": branch.id, + } + vals.update(values) + return self.env["odoo.repository.branch"].create(vals) + + def _create_odoo_module_branch(self, module, branch, **values): + vals = { + "module_id": module.id, + "branch_id": branch.id, + } + vals.update(values) + return self.env["odoo.module.branch"].create(vals) + + @classmethod + def _handle_cleanup(cls): + """Cleanup dandling git processes once tests are done. + + GitPython is spawning git processes, themselves spawning others + dettached git processes which can take few seconds to stop afterwards. + Odoo >= 17.0 is randomly WARNING about such dangling processes when + running tests (depending if they are already stopped or not), + so here we are cleaning them before Odoo gets a chance to detect them + (see 'check_remaining_processes' in 'tests.common.BaseCase'). + """ + + def kill_remaining_git_processes(): + current_process = psutil.Process() + children = current_process.children(recursive=False) + for child in children: + if child.name() != "git": + continue + _logger.info("A git process was found, killing it: %s", child) + child.kill() + psutil.wait_procs(children, timeout=10) + + cls.addClassCleanup(kill_remaining_git_processes) diff --git a/odoo_repository/tests/test_base_scanner.py b/odoo_repository/tests/test_base_scanner.py new file mode 100644 index 0000000..a3d2e51 --- /dev/null +++ b/odoo_repository/tests/test_base_scanner.py @@ -0,0 +1,214 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import tempfile + +from odoo.addons.odoo_repository.lib.scanner import BaseScanner + +from .common import Common + + +class TestBaseScanner(Common): + def _init_scanner(self, **params): + kwargs = { + "org": self.fork_org, + "name": self.repo_name, + "clone_url": self.repo_upstream_path, + "branches": [ + self.branch1_name, + self.branch2_name, + self.branch3_name, + ], + "repositories_path": self.repositories_path, + } + if params: + kwargs.update(params) + return BaseScanner(**kwargs) + + def test_init(self): + scanner = self._init_scanner() + self.assertTrue(scanner.repositories_path.exists()) + self.assertEqual(scanner.path.parts[-1], self.repo_name) + self.assertEqual(scanner.path.parts[-2], self.fork_org) + self.assertEqual(scanner.full_name, f"{self.fork_org}/{self.repo_name}") + + def test_clone_url_github_token(self): + # Without token + base_clone_url = "https://github.com/OCA/test" + scanner = self._init_scanner(repo_type="github", clone_url=base_clone_url) + self.assertEqual(scanner.clone_url, base_clone_url) + # With a token + token = "test" + scanner = self._init_scanner( + repo_type="github", clone_url=base_clone_url, token=token + ) + token_clone_url = f"https://oauth2:{token}@github.com/OCA/test" + self.assertEqual(scanner.clone_url, token_clone_url) + + def test_sync(self): + scanner = self._init_scanner(repositories_path=tempfile.mkdtemp()) + # Clone + self.assertFalse(scanner.path.exists()) + self.assertFalse(scanner.is_cloned) + scanner.sync() + self.assertTrue(scanner.is_cloned) + # Fetch once cloned + scanner.sync() + + def test_branch_exists(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + self.assertTrue(scanner._branch_exists(repo, self.branch1_name)) + self.assertTrue(scanner._branch_exists(repo, self.branch2_name)) + self.assertTrue(scanner._branch_exists(repo, self.branch3_name)) + + def test_checkout_branch(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + branch = self.branch2_name + branch_sha = repo.refs[f"origin/{branch}"].object.hexsha + self.assertNotEqual(repo.head.object.hexsha, branch_sha) + scanner._checkout_branch(repo, branch) + self.assertEqual(repo.head.object.hexsha, branch_sha) + + def test_get_last_fetched_commit(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + branch1 = self.branch1_name + branch2 = self.branch2_name + branch3 = self.branch3_name + branch1_sha = repo.refs[f"origin/{branch1}"].object.hexsha + branch2_sha = repo.refs[f"origin/{branch2}"].object.hexsha + branch3_sha = repo.refs[f"origin/{branch3}"].object.hexsha + self.assertEqual( + scanner._get_last_fetched_commit(repo, branch1), branch1_sha + ) + self.assertEqual( + scanner._get_last_fetched_commit(repo, branch2), branch2_sha + ) + self.assertEqual( + scanner._get_last_fetched_commit(repo, branch3), branch3_sha + ) + + def test_get_module_paths(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + branch = self.branch1_name + module_paths = scanner._get_module_paths(repo, ".", branch) + self.assertEqual(len(module_paths), 1) + self.assertEqual(module_paths[0], self.addon) + + def test_get_module_paths_updated(self): + scanner = self._init_scanner() + scanner.sync() + branch = self.branch1_name + with scanner.repo() as repo: + initial_commit = scanner._get_last_fetched_commit(repo, branch) + # Case where from_commit and to_commit are the same: no change detected + module_paths = scanner._get_module_paths_updated( + repo, + relative_path=".", + from_commit=initial_commit, + to_commit=initial_commit, + branch=branch, + ) + self.assertFalse(module_paths) + # Update the upstream repository with a new commit + self._update_module_version_on_branch(branch, "1.0.1") + # Module is now detected has updated + scanner.sync() # Fetch new commit from upstream repo + with scanner.repo() as repo: + last_commit = scanner._get_last_fetched_commit(repo, branch) + module_paths = scanner._get_module_paths_updated( + repo, + relative_path=".", + from_commit=initial_commit, + to_commit=last_commit, + branch=branch, + ) + self.assertEqual(len(module_paths), 1) + self.assertEqual(module_paths.pop(), self.addon) + + def test_filter_file_path(self): + scanner = self._init_scanner() + self.assertFalse(scanner._filter_file_path("fr.po")) + self.assertTrue(scanner._filter_file_path("test.py")) + + def test_get_last_commit_of_git_tree(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + branch = self.branch1_name + remote_branch = f"origin/{branch}" + module = self.addon + module_tree = repo.tree(remote_branch) / module + all_commits = [c.hexsha for c in repo.iter_commits(remote_branch)] + commit = scanner._get_last_commit_of_git_tree(remote_branch, module_tree) + self.assertIn(commit, all_commits) + + def test_get_commits_of_git_tree(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + branch = self.branch1_name + remote_branch = f"origin/{branch}" + module = self.addon + module_tree = repo.tree(remote_branch) / module + all_commits = [c.hexsha for c in repo.iter_commits(remote_branch)] + commits = scanner._get_commits_of_git_tree( + from_=None, to_=remote_branch, tree=module_tree + ) + for commit in commits: + self.assertIn(commit, all_commits) + + def test_odoo_module(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + branch = self.branch1_name + remote_branch = f"origin/{branch}" + module = self.addon + module_tree = repo.tree(remote_branch) / module + self.assertTrue(scanner._odoo_module(module_tree)) + + def test_manifest_exists(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + branch = self.branch1_name + remote_branch = f"origin/{branch}" + # Check module tree: OK + module = self.addon + module_tree = repo.tree(remote_branch) / module + self.assertTrue(scanner._manifest_exists(module_tree)) + # Check repository root tree: KO + self.assertFalse(scanner._manifest_exists(repo.tree(remote_branch))) + + def test_get_subtree(self): + scanner = self._init_scanner() + scanner.sync() + with scanner.repo() as repo: + branch = self.branch1_name + remote_branch = f"origin/{branch}" + module = self.addon + # Module/folder exists: OK + self.assertTrue(scanner._get_subtree(repo.tree(remote_branch), module)) + # Module/folder doesn't exist: KO + self.assertFalse(scanner._get_subtree(repo.tree(remote_branch), "none")) + + def test_workaround_fs_errors(self): + scanner = self._init_scanner( + repositories_path=tempfile.mkdtemp(), + workaround_fs_errors=True, + ) + # Clone + self.assertFalse(scanner.path.exists()) + self.assertFalse(scanner.is_cloned) + scanner.sync() + self.assertTrue(scanner.is_cloned) + # Fetch once cloned + scanner.sync() diff --git a/odoo_repository/tests/test_odoo_module_branch.py b/odoo_repository/tests/test_odoo_module_branch.py new file mode 100644 index 0000000..c0424d7 --- /dev/null +++ b/odoo_repository/tests/test_odoo_module_branch.py @@ -0,0 +1,97 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo.exceptions import ValidationError + +from .common import Common + + +class TestOdooModuleBranch(Common): + def test_constraint_generic_depends_on_specific(self): + generic_mod = self._create_odoo_module("generic_mod") + generic_mod_branch = self._create_odoo_module_branch( + generic_mod, self.branch, specific=False + ) + specific_mod = self._create_odoo_module("specific_mod") + specific_mod_branch = self._create_odoo_module_branch( + specific_mod, self.branch, specific=True + ) + with self.assertRaises(ValidationError): + generic_mod_branch.dependency_ids = specific_mod_branch + + def test_dependency_level(self): + # base module in the dependencies tree + mod_base = self._create_odoo_module("base") + mod_base_branch = self._create_odoo_module_branch( + mod_base, self.branch, is_standard=True + ) + self.assertEqual(mod_base_branch.global_dependency_level, 1) + self.assertEqual(mod_base_branch.non_std_dependency_level, 0) + # add a standard module depending on the base one + mod_std = self._create_odoo_module("std") + mod_std_branch = self._create_odoo_module_branch( + mod_std, + self.branch, + is_standard=True, + dependency_ids=[(4, mod_base_branch.id)], + ) + self.assertEqual(mod_std_branch.global_dependency_level, 2) + self.assertEqual(mod_std_branch.non_std_dependency_level, 0) + # add a non-standard module depending on the std one + mod_non_std = self._create_odoo_module("non_std") + mod_non_std_branch = self._create_odoo_module_branch( + mod_non_std, + self.branch, + is_standard=False, + dependency_ids=[(4, mod_std_branch.id)], + ) + self.assertEqual(mod_non_std_branch.global_dependency_level, 3) + self.assertEqual(mod_non_std_branch.non_std_dependency_level, 1) + # add another one depending on base module + mod_non_std2 = self._create_odoo_module("non_std2") + mod_non_std2_branch = self._create_odoo_module_branch( + mod_non_std2, + self.branch, + is_standard=False, + dependency_ids=[(4, mod_base_branch.id)], + ) + self.assertEqual(mod_non_std2_branch.global_dependency_level, 2) + self.assertEqual(mod_non_std2_branch.non_std_dependency_level, 1) + # add another one depending on non-std module + mod_non_std3 = self._create_odoo_module("non_std3") + mod_non_std3_branch = self._create_odoo_module_branch( + mod_non_std3, + self.branch, + is_standard=False, + dependency_ids=[(4, mod_non_std_branch.id)], + ) + self.assertEqual(mod_non_std3_branch.global_dependency_level, 4) + self.assertEqual(mod_non_std3_branch.non_std_dependency_level, 2) + + def test_find(self): + mb_model = self.env["odoo.module.branch"] + mod = self._create_odoo_module("my_module") + repo = self.odoo_repository + repo2 = self.odoo_repository.copy({"name": "Repo2"}) + # Find orphaned module + mod_orphaned = mb_model._create_orphaned_module_branch(self.branch, mod) + self.assertEqual(mb_model._find(self.branch, mod, repo), mod_orphaned) + # Find generic module + repo_branch = self._create_odoo_repository_branch(repo, self.branch) + mod_generic = self._create_odoo_module_branch( + mod, + self.branch, + specific=False, + repository_branch_id=repo_branch.id, + ) + self.assertEqual(mb_model._find(self.branch, mod, repo), mod_generic) + # Find module in current repository + repo2_branch = self._create_odoo_repository_branch(repo2, self.branch) + mod_in_repo2 = self._create_odoo_module_branch( + mod, + self.branch, + repository_branch_id=repo2_branch.id, + ) + self.assertEqual(mb_model._find(self.branch, mod, repo2), mod_in_repo2) + # While we have 3 modules (hosted in different repos or orphaned) + self.assertEqual(len(mod.module_branch_ids), 3) diff --git a/odoo_repository/tests/test_odoo_repository_scan.py b/odoo_repository/tests/test_odoo_repository_scan.py new file mode 100644 index 0000000..9298124 --- /dev/null +++ b/odoo_repository/tests/test_odoo_repository_scan.py @@ -0,0 +1,244 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo import fields + +from .common import Common + + +class TestOdooRepositoryScan(Common): + def test_check_config(self): + self.odoo_repository._check_config() + + def test_action_scan_basic(self): + """Test the creation of a module when scanning a (generic) repository.""" + self.assertFalse(self.odoo_repository.specific) + module = self.env["odoo.module"].search([("name", "=", self.module_name)]) + self.assertFalse(module) + self._run_odoo_repository_action_scan(self.branch.id) + # Check module technical name + module = self.env["odoo.module"].search([("name", "=", self.module_name)]) + self.assertTrue(module) + # Check module branch + module_branch = self.env["odoo.module.branch"].search( + [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] + ) + self.assertEqual(module_branch.module_name, self.module_name) + self.assertTrue(module_branch.last_scanned_commit) + self.assertEqual(module_branch.repository_id, self.odoo_repository) + self.assertEqual(module_branch.org_id, self.org) + self.assertEqual(module_branch.title, "Test") + self.assertEqual(module_branch.category_id.name, "Test Module") + self.assertItemsEqual( + module_branch.author_ids.mapped("name"), + ["Odoo Community Association (OCA)", "Camptocamp"], + ) + self.assertFalse(module_branch.specific) + self.assertEqual(module_branch.dependency_ids.module_name, "base") + self.assertFalse(module_branch.dependency_ids.specific) + self.assertEqual(module_branch.license_id.name, "AGPL-3") + self.assertEqual(module_branch.version, "1.0.0") + self.assertEqual(module_branch.version_ids.manifest_value, "1.0.0") + self.assertEqual(module_branch.version_ids.name, f"{self.branch.name}.1.0.0") + self.assertEqual( + module_branch.version_ids.commit, module_branch.last_scanned_commit + ) + self.assertFalse(module_branch.version_ids.has_migration_script) + self.assertTrue(module_branch.sloc_python) + self.assertEqual(module_branch.addons_path, ".") + # Check repository branch + repo_branch = module_branch.repository_branch_id + self.assertEqual(repo_branch.branch_id, self.branch) + self.assertEqual( + repo_branch.last_scanned_commit, module_branch.last_scanned_commit + ) + + def test_action_scan_repo_specific(self): + """Test the creation of a module when scanning a specific repository.""" + self.odoo_repository.specific = True + self.odoo_repository.write( + { + "specific": True, + "branch_ids": [ + fields.Command.create({"branch_id": self.branch.id}), + ], + } + ) + self.assertTrue(self.odoo_repository.specific) + self._run_odoo_repository_action_scan(self.branch.id) + # Check module data + module = self.env["odoo.module"].search([("name", "=", self.module_name)]) + module_branch = self.env["odoo.module.branch"].search( + [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] + ) + self.assertTrue(module_branch.specific) + self.assertFalse(module_branch.dependency_ids.specific) + + def test_action_scan_repo_module_exists(self): + """Test the update of existing module when scanning a repository. + + If a module has already been scanned in the current repository, a second + scan will trigger an update of its data. + """ + # First scan, like in `test_action_scan_first_time` + self._run_odoo_repository_action_scan(self.branch.id) + module = self.env["odoo.module"].search([("name", "=", self.module_name)]) + module_branch = self.env["odoo.module.branch"].search( + [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] + ) + # Change some data in the module before triggering the second scan + module_branch.write({"title": False}) + # Launch a second scan (force it to make it happen) + self._run_odoo_repository_action_scan(self.branch.id, force=True) + self.assertEqual(module_branch.title, "Test") + + def test_action_scan_orphaned_module_exists(self): + """Test the link of an orphaned module when scanning a repository. + + An orphaned module is a record without repository assigned while we know + its name and branch. Such record is automatically created when generating + dependencies of a given module that have not been scanned yet, or by + importing installed modules in a project (see `odoo_project` Odoo module). + + If such a module matches a module scanned in a repository, it is updated + accordingly to belong to this repository. + """ + # Create an orphaned module. + # To ease its creation, we run a scan to get the record created, and + # we update it to make it orphaned. + self._run_odoo_repository_action_scan(self.branch.id) + module = self.env["odoo.module"].search([("name", "=", self.module_name)]) + module_branch = self.env["odoo.module.branch"].search( + [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] + ) + module_branch.write( + { + "repository_branch_id": False, + "last_scanned_commit": False, + "dependency_ids": False, + "version_ids": False, + } + ) + # Launch a scan + self._run_odoo_repository_action_scan(self.branch.id, force=True) + self.assertEqual(module_branch.repository_id, self.odoo_repository) + + def _create_wrong_repo_branch(self, repo_sequence=100): + wrong_repo = self.env["odoo.repository"].create( + { + "org_id": self.odoo_repository.org_id.id, + "name": "wrong_repo", + "repo_url": "https://example.net/OCA-test/wrong_repo", + "clone_url": "https://example.net/OCA-test/wrong_repo", + "repo_type": "github", + "sequence": repo_sequence, + } + ) + wrong_repo_branch = self.env["odoo.repository.branch"].create( + { + "repository_id": wrong_repo.id, + "branch_id": self.branch.id, + } + ) + return wrong_repo_branch + + def _create_unmerged_module_branch(self): + # To ease the creation of such module, we run a scan to get the record + # created, and we update it to make it unmerged/pending. + self._run_odoo_repository_action_scan(self.branch.id) + module = self.env["odoo.module"].search([("name", "=", self.module_name)]) + module_branch = self.env["odoo.module.branch"].search( + [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] + ) + wrong_repo_branch = self._create_wrong_repo_branch( + # Lower priority than self.odoo_repository + repo_sequence=200 + ) + wrong_repo = wrong_repo_branch.repository_id + module_branch.write( + { + "repository_branch_id": wrong_repo_branch, + "last_scanned_commit": False, + "dependency_ids": False, + "version_ids": False, + "pr_url": f"{wrong_repo.repo_url}/pull/1", + "specific": False, + } + ) + return module_branch + + def test_action_scan_repo_generic_unmerged_module_exists(self): + """Test link of an unmerged module when scanning a generic repository. + + An unmerged module is like an orphaned module but with a PR attached. + However such PR indicates from which repository a module is coming from, + but this information could also be wrong (wrong PR detected on the wrong + repository). + + Note: unmerged modules can only be generic, as PR detection is restricted + only to generic modules. + + When scanning a generic repository, if an unmerged module is + detected it should be linked to this scanned repository. + """ + self.odoo_repository.specific = False + # Create an unmerged module + module_branch = self._create_unmerged_module_branch() + self.assertFalse(module_branch.specific) + self.assertNotEqual(module_branch.repository_id, self.odoo_repository) + # Launch a scan + self._run_odoo_repository_action_scan(self.branch.id, force=True) + self.assertFalse(module_branch.specific) + self.assertEqual(module_branch.repository_id, self.odoo_repository) + + def test_action_scan_repo_specific_unmerged_module_exists(self): + """Test non-link of an unmerged module when scanning a specific repository. + + When scanning a specific repository, detection of unmerged modules is + not done as such modules are linked to generic repositories only. + """ + self.odoo_repository.specific = True + # Create an unmerged module + module_branch = self._create_unmerged_module_branch() + self.assertFalse(module_branch.specific) + self.assertNotEqual(module_branch.repository_id, self.odoo_repository) + # Launch a scan + self._run_odoo_repository_action_scan(self.branch.id, force=True) + # Unmerged module hasn't been attached to the scanned repository + self.assertNotEqual(module_branch.repository_id, self.odoo_repository) + + def test_action_scan_uninstallable_module(self): + """Test scan of an 'installable: False' module. + + Such module should not be created with its dependencies (Odoo, Python...) + or versions history to not pollute the DB. Such data could be + outdated as the module is flagged as not installable. They will be updated + once the module is migrated/installable. + """ + self._update_module_installable_on_branch(self.branch.name, installable=False) + module = self.env["odoo.module"].search([("name", "=", self.module_name)]) + self.assertFalse(module) + self._run_odoo_repository_action_scan(self.branch.id) + module = self.env["odoo.module"].search([("name", "=", self.module_name)]) + self.assertTrue(module) + # Check module branch + module_branch = self.env["odoo.module.branch"].search( + [("module_id", "=", module.id), ("branch_id", "=", self.branch.id)] + ) + self.assertEqual(module_branch.module_name, self.module_name) + self.assertTrue(module_branch.last_scanned_commit) + self.assertEqual(module_branch.repository_id, self.odoo_repository) + self.assertEqual(module_branch.org_id, self.org) + self.assertEqual(module_branch.title, "Test") + self.assertEqual(module_branch.category_id.name, "Test Module") + self.assertItemsEqual( + module_branch.author_ids.mapped("name"), + ["Odoo Community Association (OCA)", "Camptocamp"], + ) + self.assertFalse(module_branch.specific) + # No dependencies + self.assertFalse(module_branch.dependency_ids) + self.assertFalse(module_branch.external_dependencies) + self.assertFalse(module_branch.python_dependency_ids) + # No version scanned + self.assertFalse(module_branch.version_ids) diff --git a/odoo_repository/tests/test_repository_scanner.py b/odoo_repository/tests/test_repository_scanner.py new file mode 100644 index 0000000..7102b54 --- /dev/null +++ b/odoo_repository/tests/test_repository_scanner.py @@ -0,0 +1,215 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo.addons.odoo_repository.utils.scanner import RepositoryScannerOdooEnv + +from .common import Common + + +class TestRepositoryScanner(Common): + def _init_scanner(self, **params): + kwargs = { + "org": self.org.name, + "name": self.repo_name, + "clone_url": self.repo_upstream_path, + "version": self.branch.name, + "branch": self.branch.name, + "addons_paths_data": [ + { + "relative_path": ".", + "is_standard": False, + "is_enterprise": False, + "is_community": True, + }, + ], + "repositories_path": self.repositories_path, + "env": self.env, + } + if params: + kwargs.update(params) + return RepositoryScannerOdooEnv(**kwargs) + + def test_init(self): + scanner = self._init_scanner() + self.assertTrue(scanner.repositories_path.exists()) + self.assertEqual(scanner.path.parts[-1], self.repo_name) + self.assertEqual(scanner.path.parts[-2], self.fork_org) + self.assertEqual(scanner.full_name, f"{self.fork_org}/{self.repo_name}") + + def test_sync(self): + scanner = self._init_scanner() + scanner.sync() + + def test_get_odoo_repository_id(self): + scanner = self._init_scanner() + repo_id = scanner._get_odoo_repository_id() + self.assertEqual(repo_id, self.odoo_repository.id) + + def test_get_odoo_branch_id(self): + scanner = self._init_scanner() + branch_id = scanner._get_odoo_branch_id(self.branch.name) + self.assertEqual(branch_id, self.branch.id) + + def test_create_odoo_repository_branch(self): + scanner = self._init_scanner() + repo_id = scanner._get_odoo_repository_id() + branch_id = scanner._get_odoo_branch_id(self.branch.name) + # The repository branch doesn't exist yet + expected_repo_branch_id = scanner._get_odoo_repository_branch_id( + repo_id, branch_id + ) + self.assertFalse(expected_repo_branch_id) + # Create it + repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) + self.assertTrue(repo_branch_id) + self.assertEqual( + repo_branch_id, scanner._get_odoo_repository_branch_id(repo_id, branch_id) + ) + + def test_get_repo_last_scanned_commit(self): + scanner = self._init_scanner() + repo_id = scanner._get_odoo_repository_id() + branch_id = scanner._get_odoo_branch_id(self.branch.name) + repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) + repo_branch = self.env["odoo.repository.branch"].browse(repo_branch_id) + # Nothing has been scanned until now + self.assertFalse(scanner._get_repo_last_scanned_commit(repo_branch_id)) + # Clone/fetch the repo + scanner.sync() + with scanner.repo() as repo: + last_fetched_commit = scanner._get_last_fetched_commit( + repo, self.branch.name + ) + # Simulate the end of scan + repo_branch.last_scanned_commit = last_fetched_commit + # Check again + last_scanned_commit = scanner._get_repo_last_scanned_commit(repo_branch_id) + self.assertEqual(last_fetched_commit, last_scanned_commit) + + def test_detect_modules_to_scan(self): + scanner = self._init_scanner() + scanner._clone() + repo_id = scanner._get_odoo_repository_id() + with scanner.repo() as repo: + res = scanner._detect_modules_to_scan(repo, repo_id) + self.assertTrue(res) + self.assertIn("my_module", res["addons_paths"]["."]["modules_to_scan"]) + + def test_detect_modules_to_scan_in_addons_path(self): + scanner = self._init_scanner() + scanner._clone() + with scanner.repo() as repo: + scanner._checkout_branch(repo, self.branch.name) + repo_id = scanner._get_odoo_repository_id() + branch_id = scanner._get_odoo_branch_id(self.branch.name) + repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) + last_fetched_commit = scanner._get_last_fetched_commit( + repo, self.branch.name + ) + last_scanned_commit = scanner._get_repo_last_scanned_commit(repo_branch_id) + # Scan the addons_path (root of the repository here) + modules_to_scan = scanner._detect_modules_to_scan_in_addons_path( + repo, + scanner.addons_paths_data[0]["relative_path"], + repo_branch_id, + last_fetched_commit, + last_scanned_commit, + ) + module = self.addon + self.assertIn(module, modules_to_scan) + + def test_scan_module(self): + scanner = self._init_scanner() + scanner._clone() + with scanner.repo() as repo: + scanner._checkout_branch(repo, self.branch.name) + repo_id = scanner._get_odoo_repository_id() + branch_id = scanner._get_odoo_branch_id(self.branch.name) + repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) + module_path = self.addon + remote_branch = f"origin/{self.branch.name}" + module_tree = repo.tree(remote_branch) / module_path + last_module_commit = scanner._get_last_commit_of_git_tree( + remote_branch, module_tree + ) + # Scan module + specs = scanner.addons_paths_data[0] + data = scanner._scan_module( + repo, + repo_branch_id, + module_path, + last_module_commit, + specs, + ) + self.assertTrue(data) + self.assertTrue(data["code"]) + self.assertTrue(data["manifest"]) + self.assertEqual(data["is_standard"], specs["is_standard"]) + self.assertEqual(data["is_enterprise"], specs["is_enterprise"]) + self.assertEqual(data["is_community"], specs["is_community"]) + self.assertEqual(data["last_scanned_commit"], last_module_commit) + self.assertIn("1.0.0", data["versions"]) + self.assertEqual(data["versions"]["1.0.0"]["commit"], last_module_commit) + self.assertFalse(data["versions"]["1.0.0"]["migration_script"]) + + def test_push_scanned_data(self): + scanner = self._init_scanner() + scanner._clone() + with scanner.repo() as repo: + scanner._checkout_branch(repo, self.branch.name) + repo_id = scanner._get_odoo_repository_id() + branch_id = scanner._get_odoo_branch_id(self.branch.name) + repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) + module = self.addon + remote_branch = f"origin/{self.branch.name}" + module_tree = repo.tree(remote_branch) / module + last_module_commit = scanner._get_last_commit_of_git_tree( + remote_branch, module_tree + ) + specs = scanner.addons_paths_data[0] + data = scanner._scan_module( + repo, + repo_branch_id, + module, + last_module_commit, + specs, + ) + # Push scanned data + module_branch = scanner._push_scanned_data(repo_branch_id, module, data) + self.assertEqual(module_branch.module_id.name, module) + self.assertEqual(module_branch.repository_branch_id.id, repo_branch_id) + self.assertRecordValues( + module_branch, + [ + { + "repository_branch_id": repo_branch_id, + "is_standard": specs["is_standard"], + "is_enterprise": specs["is_enterprise"], + "is_community": specs["is_community"], + "application": data["manifest"].get("application", False), + "installable": data["manifest"]["installable"], + "sloc_python": data["code"]["Python"], + "sloc_xml": data["code"]["XML"], + "sloc_js": data["code"]["JavaScript"], + "sloc_css": data["code"]["CSS"], + "last_scanned_commit": last_module_commit, + } + ], + ) + + def test_update_last_scanned_commit(self): + scanner = self._init_scanner() + scanner._clone() + repo_id = scanner._get_odoo_repository_id() + branch_id = scanner._get_odoo_branch_id(self.branch.name) + repo_branch_id = scanner._create_odoo_repository_branch(repo_id, branch_id) + repo_branch = self.env["odoo.repository.branch"].browse(repo_branch_id) + with scanner.repo() as repo: + last_repo_commit = scanner._get_last_fetched_commit(repo, self.branch.name) + self.assertFalse(repo_branch.last_scanned_commit) + scanner._update_last_scanned_commit(repo_branch_id, last_repo_commit) + self.assertEqual(repo_branch.last_scanned_commit, last_repo_commit) + + def test_workaround_fs_errors(self): + scanner = self._init_scanner(workaround_fs_errors=True) + scanner.sync() diff --git a/odoo_repository/tests/test_sync_node.py b/odoo_repository/tests/test_sync_node.py new file mode 100644 index 0000000..7b2db77 --- /dev/null +++ b/odoo_repository/tests/test_sync_node.py @@ -0,0 +1,55 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from .common import Common + + +class TestSyncNode(Common): + def test_sync_node(self): + # Scan a repository + self.odoo_repository.with_context(queue_job__no_delay=True).action_scan( + self.branch.ids + ) + # Check data to sync + data = self.env["odoo.module.branch"]._get_modules_data() + self.assertEqual(len(data), 1) # 1 module scanned + module = data[0] + self.assertEqual(module["module"], self.module_name) + self.assertEqual(module["branch"], self.branch.name) + self.assertEqual(module["version"], "1.0.0") + self.assertTrue(module["versions"]) + self.assertEqual(module["addons_path"], ".") + self.assertEqual(module["repository"]["name"], self.odoo_repository.name) + self.assertEqual(module["repository"]["org"], self.odoo_repository.org_id.name) + self.assertEqual( + module["repository"]["repo_type"], self.odoo_repository.repo_type + ) + self.assertEqual( + module["repository"]["repo_url"], self.odoo_repository.repo_url + ) + # Sync these data with a node + # NOTE as we are using the same node to sync with in tests, we change + # the content of the data to sync to create a new module + data[0].update( + module="synced", + version="2.0.0", + branch=self.branch2_name, + ) + new_module = self.env["odoo.module"].search([("name", "=", "synced")]) + self.assertFalse(new_module) + self.env["odoo.repository"]._import_data(data) + nb_modules = self.env["odoo.module"].search_count([]) + self.assertTrue(nb_modules >= 2) + new_module = self.env["odoo.module.branch"].search( + [("module_name", "=", "synced")] + ) + self.assertTrue(new_module) + self.assertEqual(new_module.version, "2.0.0") + self.assertEqual(new_module.branch_id.name, self.branch2_name) + # Existing module didn't changed + existing_module = self.env["odoo.module.branch"].search( + [("module_name", "=", self.module_name)] + ) + self.assertTrue(existing_module) + self.assertEqual(existing_module.version, "1.0.0") + self.assertEqual(existing_module.branch_id, self.branch) diff --git a/odoo_repository/tests/test_utils.py b/odoo_repository/tests/test_utils.py new file mode 100644 index 0000000..612ea72 --- /dev/null +++ b/odoo_repository/tests/test_utils.py @@ -0,0 +1,25 @@ +# Copyright 2024 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo.tests.common import TransactionCase + +from odoo.addons.odoo_repository.utils.module import adapt_version + + +class TestUtils(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.env = cls.env(context=dict(cls.env.context, tracking_disable=True)) + + def test_adapt_version(self): + # Module version equals major version: add prefix + self.assertEqual(adapt_version("14.0", "14.0"), "14.0.14.0") + # Basic module version: add prefix + self.assertEqual(adapt_version("14.0", "1.0.0"), "14.0.1.0.0") + # Module version already prefixed with major version + self.assertEqual(adapt_version("14.0", "14.0.1.0.0"), "14.0.1.0.0") + # Dot chars added as prefix or suffix in the provided version + self.assertEqual(adapt_version("14.0", ".1.0.0"), "14.0.1.0.0") + self.assertEqual(adapt_version("14.0", "1.0.0."), "14.0.1.0.0") + self.assertEqual(adapt_version("14.0", "...1.0.0..."), "14.0.1.0.0") diff --git a/odoo_repository/utils/__init__.py b/odoo_repository/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/odoo_repository/utils/github.py b/odoo_repository/utils/github.py new file mode 100644 index 0000000..083e1b6 --- /dev/null +++ b/odoo_repository/utils/github.py @@ -0,0 +1,30 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import os + +import requests + +GITHUB_URL = "https://github.com" +GITHUB_API_URL = "https://api.github.com" + + +def request(env, url, method="get", params=None, json=None): + """Request GitHub API.""" + headers = {"Accept": "application/vnd.github.groot-preview+json"} + key = "odoo_repository_github_token" + token = env["ir.config_parameter"].sudo().get_param(key, "") or os.environ.get( + "GITHUB_TOKEN" + ) + if token: + headers.update({"Authorization": f"token {token}"}) + full_url = "/".join([GITHUB_API_URL, url]) + kwargs = {"headers": headers} + if json: + kwargs.update(json=json) + if params: + kwargs.update(params=params) + response = getattr(requests, method)(full_url, **kwargs) + if not response.ok: + raise RuntimeError(response.text) + return response.json() diff --git a/odoo_repository/utils/module.py b/odoo_repository/utils/module.py new file mode 100644 index 0000000..a20c827 --- /dev/null +++ b/odoo_repository/utils/module.py @@ -0,0 +1,25 @@ +# Copyright 2023 Camptocamp SA +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl) + + +def adapt_version(major_version: str, module_version: str): + """Return the module version prefixed with major version. + + This function has been copied and adapted from upstream Odoo source code: + https://github.com/odoo/odoo/blob/16.0/odoo/modules/module.py#L527-L533 + """ + # Remove extra special characters (e.g. '14.0.' => '14.0') + # NOTE: we need to sanitize such versions in order to compute a + # sequence number helping the sort of module versions. + chars = ["."] + for char in chars: + while module_version.startswith(char): + module_version = module_version[1:] + while module_version.endswith(char): + module_version = module_version[:-1] + # Append major Odoo version as prefix if needed + if module_version == major_version or not module_version.startswith( + major_version + "." + ): + module_version = f"{major_version}.{module_version}" + return module_version diff --git a/odoo_repository/utils/scanner.py b/odoo_repository/utils/scanner.py new file mode 100644 index 0000000..954b80e --- /dev/null +++ b/odoo_repository/utils/scanner.py @@ -0,0 +1,90 @@ +# Copyright 2023 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +from odoo.tools import config + +from ..lib.scanner import RepositoryScanner + + +class RepositoryScannerOdooEnv(RepositoryScanner): + """RepositoryScanner running on the same server than Odoo. + + This class takes an additional `env` parameter (`odoo.api.Environment`) + used to request Odoo, and implement required methods to use it. + """ + + def __init__(self, *args, **kwargs): + if kwargs.get("env"): + self.env = kwargs.pop("env") + super().__init__(*args, **kwargs) + + def _get_odoo_repository_id(self): + return ( + self.env["odoo.repository"] + .search([("name", "=", self.name), ("org_id", "=", self.org)]) + .id + ) + + def _get_odoo_branch_id(self, version): + return self.env["odoo.branch"].search([("name", "=", version)]).id + + def _get_odoo_repository_branch_id(self, repo_id, branch_id): + args = [ + ("repository_id", "=", repo_id), + ("branch_id", "=", branch_id), + ] + repo_branch = self.env["odoo.repository.branch"].search(args, limit=1) + if repo_branch: + return repo_branch.id + + def _create_odoo_repository_branch(self, repo_id, branch_id, cloned_branch=None): + repo_branch_id = self._get_odoo_repository_branch_id(repo_id, branch_id) + if not repo_branch_id: + values = { + "repository_id": repo_id, + "branch_id": branch_id, + } + if cloned_branch: + values["cloned_branch"] = cloned_branch + repo_branch_model = self.env["odoo.repository.branch"] + repo_branch_id = repo_branch_model.create(values).id + return repo_branch_id + + def _get_repo_last_scanned_commit(self, repo_branch_id): + repo_branch_model = self.env["odoo.repository.branch"] + repo_branch = repo_branch_model.browse(repo_branch_id) + return repo_branch.last_scanned_commit + + def _is_module_blacklisted(self, module): + return bool( + self.env["odoo.module"].search_count( + [("name", "=", module), ("blacklisted", "=", True)] + ) + ) + + def _get_module_last_scanned_commit(self, repo_branch_id, module_name): + module_branch_model = self.env["odoo.module.branch"] + args = [ + ("repository_branch_id", "=", repo_branch_id), + ("module_name", "=", module_name), + ] + module = module_branch_model.search(args) + return module.last_scanned_commit + + def _push_scanned_data(self, repo_branch_id, module, data): + res = self.env["odoo.module.branch"].push_scanned_data( + repo_branch_id, module, data + ) + # Commit after each module + if not config["test_enable"]: + self.env.cr.commit() # pylint: disable=invalid-commit + return res + + def _update_last_scanned_commit(self, repo_branch_id, last_fetched_commit): + repo_branch_model = self.env["odoo.repository.branch"] + repo_branch = repo_branch_model.browse(repo_branch_id) + repo_branch.last_scanned_commit = last_fetched_commit + # Commit after each repository/branch + if not config["test_enable"]: + self.env.cr.commit() # pylint: disable=invalid-commit + return True diff --git a/odoo_repository/views/authentication_token.xml b/odoo_repository/views/authentication_token.xml new file mode 100644 index 0000000..2cc1f4d --- /dev/null +++ b/odoo_repository/views/authentication_token.xml @@ -0,0 +1,61 @@ + + + + + authentication.token.form + authentication.token + + +
+ + + + + + +
+ + + + + authentication.token.tree + authentication.token + + + + + + + + + authentication.token.search + authentication.token + search + + + + + + + + + Authentication Tokens + ir.actions.act_window + authentication.token + + + + + + diff --git a/odoo_repository/views/menu.xml b/odoo_repository/views/menu.xml new file mode 100644 index 0000000..ec5d230 --- /dev/null +++ b/odoo_repository/views/menu.xml @@ -0,0 +1,36 @@ + + + + + + + + + diff --git a/odoo_repository/views/odoo_author.xml b/odoo_repository/views/odoo_author.xml new file mode 100644 index 0000000..48416ff --- /dev/null +++ b/odoo_repository/views/odoo_author.xml @@ -0,0 +1,52 @@ + + + + + odoo.author.form + odoo.author + +
+ + + + + +
+ + + + + odoo.author.tree + odoo.author + + + + + + + + + odoo.author.search + odoo.author + search + + + + + + + + + Authors + ir.actions.act_window + odoo.author + + + + + diff --git a/odoo_repository/views/odoo_branch.xml b/odoo_repository/views/odoo_branch.xml new file mode 100644 index 0000000..52f2904 --- /dev/null +++ b/odoo_repository/views/odoo_branch.xml @@ -0,0 +1,83 @@ + + + + + odoo.branch.form + odoo.branch + +
+
+
+ + + + + + + + + + + + + + +
+ + + + + odoo.branch.tree + odoo.branch + + + + + + + + + odoo.branch.search + odoo.branch + search + + + + + + + + + + Versions + ir.actions.act_window + odoo.branch + + + + + diff --git a/odoo_repository/views/odoo_license.xml b/odoo_repository/views/odoo_license.xml new file mode 100644 index 0000000..93c9ce6 --- /dev/null +++ b/odoo_repository/views/odoo_license.xml @@ -0,0 +1,38 @@ + + + + + odoo.license.tree + odoo.license + + + + + + + + + odoo.license.search + odoo.license + search + + + + + + + + + Licenses + ir.actions.act_window + odoo.license + + + + + diff --git a/odoo_repository/views/odoo_maintainer.xml b/odoo_repository/views/odoo_maintainer.xml new file mode 100644 index 0000000..9b84847 --- /dev/null +++ b/odoo_repository/views/odoo_maintainer.xml @@ -0,0 +1,67 @@ + + + + + odoo.maintainer.form + odoo.maintainer + +
+ + + + + + + + + + + + + + + + + + + +
+ + + + + odoo.maintainer.tree + odoo.maintainer + + + + + + + + + + odoo.maintainer.search + odoo.maintainer + search + + + + + + + + + Maintainers + ir.actions.act_window + odoo.maintainer + + + + + diff --git a/odoo_repository/views/odoo_module.xml b/odoo_repository/views/odoo_module.xml new file mode 100644 index 0000000..d59e4b2 --- /dev/null +++ b/odoo_repository/views/odoo_module.xml @@ -0,0 +1,88 @@ + + + + + odoo.module.form + odoo.module + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + + odoo.module.tree + odoo.module + + + + + + + + + + odoo.module.search + odoo.module + search + + + + + + + + + + Technical Module Names + ir.actions.act_window + odoo.module + + {"default_blacklisted": 1} + + + + diff --git a/odoo_repository/views/odoo_module_branch.xml b/odoo_repository/views/odoo_module_branch.xml new file mode 100644 index 0000000..94e0842 --- /dev/null +++ b/odoo_repository/views/odoo_module_branch.xml @@ -0,0 +1,396 @@ + + + + + odoo.module.branch.form + odoo.module.branch + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + odoo.module.branch.tree + odoo.module.branch + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + odoo.module.branch.tree.recursive_dependencies + odoo.module.branch + + primary + 100 + + + global_dependency_level, non_std_dependency_level, module_name + + + show + + + show + + + show + + + + + + odoo.module.branch.search + odoo.module.branch + search + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + odoo.module.branch.pivot + odoo.module.branch + + + + + + + + + + odoo.module.branch.graph + odoo.module.branch + + + + + + + + + Modules + ir.actions.act_window + odoo.module.branch + list,form + + { + 'search_default_installable': 1, + 'search_default_with_repository': 2 + } + + + + Modules Analysis + ir.actions.act_window + odoo.module.branch + pivot,graph + + { + 'search_default_installable': 1, + 'search_default_with_repository': 2, + 'pivot_column_groupby': ['branch_id'], + 'pivot_row_groupby': ['org_id', 'repository_id'], + 'graph_mode': 'line', + 'graph_stacked': 0, + 'graph_measure': '__count__', + 'graph_groupbys': ['branch_id', 'org_id'], + } + + + + Dependencies + ir.actions.act_window + odoo.module.branch + + + + + + + + + diff --git a/odoo_repository/views/odoo_module_category.xml b/odoo_repository/views/odoo_module_category.xml new file mode 100644 index 0000000..ad7b4a6 --- /dev/null +++ b/odoo_repository/views/odoo_module_category.xml @@ -0,0 +1,38 @@ + + + + + odoo.module.category.tree + odoo.module.category + + + + + + + + + odoo.module.category.search + odoo.module.category + search + + + + + + + + + Categories + ir.actions.act_window + odoo.module.category + + + + + diff --git a/odoo_repository/views/odoo_module_dev_status.xml b/odoo_repository/views/odoo_module_dev_status.xml new file mode 100644 index 0000000..d4fe69c --- /dev/null +++ b/odoo_repository/views/odoo_module_dev_status.xml @@ -0,0 +1,38 @@ + + + + + odoo.module.dev.status.tree + odoo.module.dev.status + + + + + + + + + odoo.module.dev.status.search + odoo.module.dev.status + search + + + + + + + + + Development Status + ir.actions.act_window + odoo.module.dev.status + + + + + diff --git a/odoo_repository/views/odoo_python_dependency.xml b/odoo_repository/views/odoo_python_dependency.xml new file mode 100644 index 0000000..575e83a --- /dev/null +++ b/odoo_repository/views/odoo_python_dependency.xml @@ -0,0 +1,38 @@ + + + + + odoo.python.dependency.tree + odoo.python.dependency + + + + + + + + + odoo.python.dependency.search + odoo.python.dependency + search + + + + + + + + + Python Dependencies + ir.actions.act_window + odoo.python.dependency + + + + + diff --git a/odoo_repository/views/odoo_repository.xml b/odoo_repository/views/odoo_repository.xml new file mode 100644 index 0000000..a2ed22a --- /dev/null +++ b/odoo_repository/views/odoo_repository.xml @@ -0,0 +1,172 @@ + + + + + odoo.repository.form + odoo.repository + +
+
+
+ +
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + odoo.repository.tree + odoo.repository + + + + + + + + + + + + + odoo.repository.search + odoo.repository + search + + + + + + + + + + + + + + + Repositories + ir.actions.act_window + odoo.repository + + + + + diff --git a/odoo_repository/views/odoo_repository_addons_path.xml b/odoo_repository/views/odoo_repository_addons_path.xml new file mode 100644 index 0000000..fe16674 --- /dev/null +++ b/odoo_repository/views/odoo_repository_addons_path.xml @@ -0,0 +1,47 @@ + + + + + odoo.repository.addons_path.form + odoo.repository.addons_path + +
+ + + + + + + + +
+ + + + + odoo.repository.addons_path.tree + odoo.repository.addons_path + + + + + + + + + + + + Addons Path + ir.actions.act_window + odoo.repository.addons_path + + + + + diff --git a/odoo_repository/views/odoo_repository_branch.xml b/odoo_repository/views/odoo_repository_branch.xml new file mode 100644 index 0000000..208b0ee --- /dev/null +++ b/odoo_repository/views/odoo_repository_branch.xml @@ -0,0 +1,63 @@ + + + + + odoo.repository.branch.form + odoo.repository.branch + +
+
+
+ + + + + + + + + + + +
+ + + + + odoo.repository.branch.tree + odoo.repository.branch + + + + + + + + + + + odoo.repository.branch.search + odoo.repository.branch + search + + + + + + + + diff --git a/odoo_repository/views/odoo_repository_org.xml b/odoo_repository/views/odoo_repository_org.xml new file mode 100644 index 0000000..fe54098 --- /dev/null +++ b/odoo_repository/views/odoo_repository_org.xml @@ -0,0 +1,52 @@ + + + + + odoo.repository.org.form + odoo.repository.org + +
+ + + + + +
+ + + + + odoo.repository.org.tree + odoo.repository.org + + + + + + + + + odoo.repository.org.search + odoo.repository.org + search + + + + + + + + + Repository Organizations + ir.actions.act_window + odoo.repository.org + + + + + diff --git a/odoo_repository/views/res_config_settings.xml b/odoo_repository/views/res_config_settings.xml new file mode 100644 index 0000000..79d43ad --- /dev/null +++ b/odoo_repository/views/res_config_settings.xml @@ -0,0 +1,79 @@ + + + + + res.config.settings.form.inherit + res.config.settings + + + + + + + + + + + + + +
+ If the environment variable GITHUB_TOKEN is set on the running system, it will be used as fallback. +
+ + + + + + + + + + + + Settings + ir.actions.act_window + res.config.settings + + form + inline + {'module' : 'odoo_repository', 'bin_size': False} + + + + diff --git a/odoo_repository/views/ssh_key.xml b/odoo_repository/views/ssh_key.xml new file mode 100644 index 0000000..47e2ede --- /dev/null +++ b/odoo_repository/views/ssh_key.xml @@ -0,0 +1,61 @@ + + + + + ssh.key.form + ssh.key + + +
+ + + + + + +
+ + + + + ssh.key.tree + ssh.key + + + + + + + + + ssh.key.search + ssh.key + search + + + + + + + + + SSH Keys + ir.actions.act_window + ssh.key + + + + + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..bc654e6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# generated from manifests external_dependencies +gitpython +odoo-addons-parser diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..6614693 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +oca-port