diff --git a/OAT.xml b/OAT.xml index 9857f5e54aca51dcabd9758b9a293a9aef196747..0be7e8a69944d5f186dbda721d82ce43b9197793 100644 --- a/OAT.xml +++ b/OAT.xml @@ -69,10 +69,14 @@ Note:If the text contains special characters, please escape them according to th + + + + @@ -80,4 +84,4 @@ Note:If the text contains special characters, please escape them according to th - \ No newline at end of file + diff --git a/tools/fotff/.gitignore b/tools/fotff/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..c9ab9e9416a2460775b9776c0df04af157fba9cf --- /dev/null +++ b/tools/fotff/.gitignore @@ -0,0 +1,28 @@ +# Binaries, caches, configs and outputs for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +logs +fotff +fotff.ini +.fotff + +# xdevice default directories +config +testcases +reports +resource + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# JetBrains IDE +.idea diff --git a/tools/fotff/LICENSE b/tools/fotff/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..29f81d812f3e768fa89638d1f72920dbfd1413a8 --- /dev/null +++ b/tools/fotff/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tools/fotff/README.md b/tools/fotff/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fb295d499b153fab314776a5fc40fa2f26e03b07 --- /dev/null +++ b/tools/fotff/README.md @@ -0,0 +1,75 @@ +# fotff + +#### 介绍 + +fotff(find out the first fault)是为OpenHarmony持续集成设计的问题自动化问题分析工具。 + +为了平衡开销与收益,考虑到开发效率、资源占用等因素影响,OpenHarmony代码合入门禁(冒烟测试)只拦截部分严重基础问题(例如开机失败、关键进程崩溃、UX布局严重错乱、电话/相机基础功能不可用等)。因此,一些会影响到更细节功能、影响兼容性、系统稳定性等的问题代码将可能被合入。 + +fotff提供了一个框架,不断地对最新持续集成版本运行测试套,然后对其中失败用例进行分析:找到或生成在该用例上次通过的持续集成版本和本次失败的持续集成版本之间的所有中间版本,然后运用二分法的思想,找到出现该问题的第一个中间版本,从而给出引入该问题的代码提交。 + +#### 软件架构 + +``` +fotff +├── .fotff # 缓存等程序运行时产生的文件的存放目录 +├── logs # 日志存放目录 +├── pkg # 版本包管理的接口定义和特定开发板形态的具体实现 +├── rec # 测试结果记录和分析 +├── tester # 测试套的接口定义和调用测试框架的具体实现 +├── utils # 一些通用的类库 +├── vcs # 版本控制相关的包,比如manifest的处理,通过OpenAPI访问gitee查询信息的函数等 +├── fotff.ini # 运行需要的必要参数配置,比如指定测试套、配置构建服务器、HTTP代理等 +└── main.go # 框架入口 +``` + +#### 安装教程 + +1. 获取[GoSDK](https://golang.google.cn/dl/)并按照指引安装。 +2. 在代码工程根目录执行```go build```编译。如下载依赖库出现网络问题,必要时配置GOPROXY代理。 +3. 更改fotff.ini,按功能需要,选择版本包和测试套的具体实现,完成对应参数配置,并将可能涉及到的测试用例集、脚本、刷机工具等放置到对应位置。 + +#### 使用说明 + +###### 普通模式 + +example: ```fotff``` + +1. 配置好fotff.ini文件后,不指定任何命令行参数直接执行二进制,即进入普通模式。此模式下,框架会自动不断地获取最新持续集成版本,并对其运行测试套,然后对其中失败用例进行分析。 +2. 分析结果在.fotff/records.json文件中记录;如果配置了邮箱信息,会发送结果到指定邮箱。 + +###### 对单个用例在指定区间内查找 + +example: ```fotff run -s pkgDir1 -f pkgDir2 -t TEST_CASE_001``` + +1. 配置好fotff.ini文件后,通过-s/-f/-t参数在命令行中分别指定成功版本/失败版本/测试用例名,即可对单个用例在指定区间内查找。此模式下,仅在指定的两个版本间进行二分查找,运行指定的运行测试用例。 +2. 分析结果在控制台中打印,不会发送邮件。 + +###### 烧写指定版本包 + +example: ```fotff flash -p pkgDir -d 7001005458323933328a01fce1dc3800``` + +配置好fotff.ini文件后,可以指定版本包目录烧写对应版本。 + +###### tips + +1. 刷机、测试具体实现可能涉及到[hdc_std](https://gitee.com/openharmony/developtools_hdc)、[xdevice](https://gitee.com/openharmony/testfwk_xdevice),安装和配置请参考对应工具的相关页面。 +2. xdevice运行需要Python运行环境,请提前安装。 +3. 刷机、测试过程需要对应开发板的驱动程序,请提前安装。 + +#### 参与贡献 + +1. Fork 本仓库 +2. 新建 Feat_xxx 分支 +3. 提交代码 +4. 新建 Pull Request + +#### 相关链接 + +[OpenHarmony CI](http://ci.openharmony.cn/dailys/dailybuilds) + +[developtools_hdc](https://gitee.com/openharmony/developtools_hdc) + +[dayu200_tools](https://gitee.com/hihope_iot/docs/tree/master/HiHope_DAYU200/烧写工具及指南) + +[testfwk_xdevice](https://gitee.com/openharmony/testfwk_xdevice) diff --git a/tools/fotff/go.mod b/tools/fotff/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..ab14fc53d891dd8f518773d74569802ce97fc411 --- /dev/null +++ b/tools/fotff/go.mod @@ -0,0 +1,28 @@ +module fotff + +go 1.19 + +require ( + code.cloudfoundry.org/archiver v0.0.0-20221114120234-625eff81a7ef + github.com/Unknwon/goconfig v1.0.0 + github.com/huandu/go-clone v1.4.1 + github.com/jedib0t/go-pretty/v6 v6.4.3 + github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/pkg/sftp v1.13.5 + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/cobra v1.6.1 + golang.org/x/crypto v0.3.0 + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df +) + +require ( + github.com/cyphar/filepath-securejoin v0.2.3 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/smartystreets/goconvey v1.7.2 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/sys v0.2.0 // indirect + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect +) diff --git a/tools/fotff/go.sum b/tools/fotff/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..9649c260cc6bb63a77b905ee0414c29cab54841f --- /dev/null +++ b/tools/fotff/go.sum @@ -0,0 +1,91 @@ +code.cloudfoundry.org/archiver v0.0.0-20221114120234-625eff81a7ef h1:YMr8OebAw8ufxTyTLPFbMmiChH4M+1RaIpsdLKojZ48= +code.cloudfoundry.org/archiver v0.0.0-20221114120234-625eff81a7ef/go.mod h1:WK8AWnIZ1W1EpPoVLzsSshXKKqP1Nzk6SoVRxD9cx54= +github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= +github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/huandu/go-assert v1.1.5 h1:fjemmA7sSfYHJD7CUqs9qTwwfdNAx7/j2/ZlHXzNB3c= +github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0JrPVhn/06U= +github.com/huandu/go-clone v1.4.1 h1:QQYjiLadyxOvdwgZoH8f1xGkvvf4+Cm8be7fo9W2QQA= +github.com/huandu/go-clone v1.4.1/go.mod h1:ReGivhG6op3GYr+UY3lS6mxjKp7MIGTknuU5TbTVaXE= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jedib0t/go-pretty/v6 v6.4.3 h1:2n9BZ0YQiXGESUSR+6FLg0WWWE80u+mIz35f0uHWcIE= +github.com/jedib0t/go-pretty/v6 v6.4.3/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= +github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go= +github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.4 h1:wZRexSlwd7ZXfKINDLsO4r7WBt3gTKONc6K/VesHvHM= +github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= +gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= +gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/fotff/main.go b/tools/fotff/main.go new file mode 100644 index 0000000000000000000000000000000000000000..2c7d73d91b11772867bb1410175252d034317594 --- /dev/null +++ b/tools/fotff/main.go @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "context" + "fotff/pkg" + "fotff/pkg/dayu200" + "fotff/pkg/mock" + "fotff/rec" + "fotff/res" + "fotff/tester" + "fotff/tester/manual" + testermock "fotff/tester/mock" + "fotff/tester/smoke" + "fotff/tester/xdevice" + "fotff/utils" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "os" + "path/filepath" +) + +var newPkgMgrFuncs = map[string]pkg.NewFunc{ + "mock": mock.NewManager, + "dayu200": dayu200.NewManager, +} + +var newTesterFuncs = map[string]tester.NewFunc{ + "mock": testermock.NewTester, + "manual": manual.NewTester, + "xdevice": xdevice.NewTester, + "smoke": smoke.NewTester, +} + +var rootCmd *cobra.Command + +func init() { + m, t := initExecutor() + rootCmd = &cobra.Command{ + Run: func(cmd *cobra.Command, args []string) { + loop(m, t) + }, + } + runCmd := initRunCmd(m, t) + flashCmd := initFlashCmd(m) + rootCmd.AddCommand(runCmd, flashCmd) +} + +func initRunCmd(m pkg.Manager, t tester.Tester) *cobra.Command { + var success, fail, testcase string + runCmd := &cobra.Command{ + Use: "run", + Short: "bin-search in (success, fail] by do given testcase to find out the fist fail, and print the corresponding issue", + RunE: func(cmd *cobra.Command, args []string) error { + return fotff(m, t, success, fail, testcase) + }, + } + runCmd.PersistentFlags().StringVarP(&success, "success", "s", "", "success package directory") + runCmd.PersistentFlags().StringVarP(&fail, "fail", "f", "", "fail package directory") + runCmd.PersistentFlags().StringVarP(&testcase, "testcase", "t", "", "testcase name") + runCmd.MarkPersistentFlagRequired("success") + runCmd.MarkPersistentFlagRequired("fail") + runCmd.MarkPersistentFlagRequired("testcase") + return runCmd +} + +func initFlashCmd(m pkg.Manager) *cobra.Command { + var flashPkg, device string + flashCmd := &cobra.Command{ + Use: "flash", + Short: "flash the given package", + RunE: func(cmd *cobra.Command, args []string) error { + return m.Flash(device, flashPkg, context.TODO()) + }, + } + flashCmd.PersistentFlags().StringVarP(&flashPkg, "package", "p", "", "package directory") + flashCmd.PersistentFlags().StringVarP(&device, "device", "d", "", "device sn") + flashCmd.MarkPersistentFlagRequired("package") + return flashCmd +} + +func main() { + utils.EnablePprof() + if err := rootCmd.Execute(); err != nil { + logrus.Errorf("failed to execute: %v", err) + os.Exit(1) + } +} + +func loop(m pkg.Manager, t tester.Tester) { + data, _ := utils.ReadRuntimeData("last_handled.rec") + var curPkg = string(data) + for { + utils.ResetLogOutput() + if err := utils.WriteRuntimeData("last_handled.rec", []byte(curPkg)); err != nil { + logrus.Errorf("failed to write last_handled.rec: %v", err) + } + logrus.Info("waiting for a newer package...") + var err error + curPkg, err = m.GetNewer(curPkg) + if err != nil { + logrus.Infof("get newer package err: %v", err) + continue + } + utils.SetLogOutput(filepath.Base(curPkg)) + logrus.Infof("now flash %s...", curPkg) + device := res.GetDevice() + if err := m.Flash(device, curPkg, context.TODO()); err != nil { + logrus.Errorf("flash package dir %s err: %v", curPkg, err) + res.ReleaseDevice(device) + continue + } + if err := t.Prepare(m.PkgDir(curPkg), device, context.TODO()); err != nil { + logrus.Errorf("do test preperation for package %s err: %v", curPkg, err) + continue + } + logrus.Info("now do test suite...") + results, err := t.DoTestTask(device, context.TODO()) + if err != nil { + logrus.Errorf("do test suite for package %s err: %v", curPkg, err) + continue + } + for _, r := range results { + logrus.Infof("do test case %s at %s done, result is %v", r.TestCaseName, device, r.Status) + } + logrus.Infof("now analysis test results...") + toFotff := rec.HandleResults(t, device, curPkg, results) + res.ReleaseDevice(device) + rec.Analysis(m, t, curPkg, toFotff) + rec.Save() + rec.Report(curPkg, t.TaskName()) + } +} + +func fotff(m pkg.Manager, t tester.Tester, success, fail, testcase string) error { + issueURL, err := rec.FindOutTheFirstFail(m, t, testcase, success, fail) + if err != nil { + logrus.Errorf("failed to find out the first fail: %v", err) + return err + } + logrus.Infof("the first fail found: %v", issueURL) + return nil +} + +func initExecutor() (pkg.Manager, tester.Tester) { + //TODO load from config file + var conf = struct { + PkgManager string `key:"pkg_manager" default:"mock"` + Tester string `key:"tester" default:"mock"` + }{} + utils.ParseFromConfigFile("", &conf) + newPkgMgrFunc, ok := newPkgMgrFuncs[conf.PkgManager] + if !ok { + logrus.Panicf("no package manager found for %s", conf.PkgManager) + } + newTesterFunc, ok := newTesterFuncs[conf.Tester] + if !ok { + logrus.Panicf("no tester found for %s", conf.Tester) + } + return newPkgMgrFunc(), newTesterFunc() +} diff --git a/tools/fotff/pkg/dayu200/build.go b/tools/fotff/pkg/dayu200/build.go new file mode 100644 index 0000000000000000000000000000000000000000..0453005fdffd475ee1141139ac6ca985d0e9b3f9 --- /dev/null +++ b/tools/fotff/pkg/dayu200/build.go @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dayu200 + +import ( + "context" + "fmt" + "fotff/res" + "fotff/utils" + "github.com/sirupsen/logrus" + "os" + "path/filepath" +) + +// These commands are copied from ci project. +const ( + preCompileCMD = `rm -rf prebuilts/clang/ohos/darwin-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/windows-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/linux-x86_64/clang-480513;bash build/prebuilts_download.sh` + // compileCMD is copied from ci project and trim useless build-target 'make_test' to enhance build efficiency. + compileCMD = `echo 'start' && export NO_DEVTOOL=1 && export CCACHE_LOG_SUFFIX="dayu200-arm32" && export CCACHE_NOHASHDIR="true" && export CCACHE_SLOPPINESS="include_file_ctime" && ./build.sh --product-name rk3568 --ccache --build-target make_all --gn-args enable_notice_collection=false` + rmOutCMD = `rm -rf out` +) + +// This list is copied from ci project. Some of them are not available, has been annotated. +var imgList = []string{ + "out/rk3568/packages/phone/images/MiniLoaderAll.bin", + "out/rk3568/packages/phone/images/boot_linux.img", + "out/rk3568/packages/phone/images/parameter.txt", + "out/rk3568/packages/phone/images/system.img", + "out/rk3568/packages/phone/images/uboot.img", + "out/rk3568/packages/phone/images/userdata.img", + "out/rk3568/packages/phone/images/vendor.img", + "out/rk3568/packages/phone/images/resource.img", + "out/rk3568/packages/phone/images/config.cfg", + "out/rk3568/packages/phone/images/ramdisk.img", + // "out/rk3568/packages/phone/images/chipset.img", + "out/rk3568/packages/phone/images/sys_prod.img", + "out/rk3568/packages/phone/images/chip_prod.img", + "out/rk3568/packages/phone/images/updater.img", + // "out/rk3568/packages/phone/updater/bin/updater_binary", +} + +// pkgAvailable returns true if all necessary images are all available to flash. +func (m *Manager) pkgAvailable(pkg string) bool { + for _, img := range imgList { + imgName := filepath.Base(img) + if _, err := os.Stat(filepath.Join(m.Workspace, pkg, imgName)); err != nil { + return false + } + } + return true +} + +// build obtain an available server, download corresponding codes, and run compile commands +// to build the corresponding package images, then transfer these images to the 'pkg' directory. +func (m *Manager) build(pkg string, rm bool, ctx context.Context) error { + logrus.Infof("now build %s", pkg) + server := res.GetBuildServer() + defer res.ReleaseBuildServer(server) + cmd := fmt.Sprintf("mkdir -p %s && cd %s && repo init -u https://gitee.com/openharmony/manifest.git", server.WorkSpace, server.WorkSpace) + if err := utils.RunCmdViaSSHContext(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { + return fmt.Errorf("remote: mkdir error: %w", err) + } + if err := utils.TransFileViaSSH(utils.Upload, server.Addr, server.User, server.Passwd, + fmt.Sprintf("%s/.repo/manifest.xml", server.WorkSpace), filepath.Join(m.Workspace, pkg, "manifest_tag.xml")); err != nil { + return fmt.Errorf("upload and replace manifest error: %w", err) + } + // 'git lfs install' may fail due to some git hooks. Call 'git lfs update --force' before install to avoid this situation. + cmd = fmt.Sprintf("cd %s && repo sync -c --no-tags --force-remove-dirty && repo forall -c 'git reset --hard && git clean -dfx && git lfs update --force && git lfs install && git lfs pull'", server.WorkSpace) + if err := utils.RunCmdViaSSHContext(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { + return fmt.Errorf("remote: repo sync error: %w", err) + } + cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, preCompileCMD) + if err := utils.RunCmdViaSSHContextNoRetry(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { + return fmt.Errorf("remote: pre-compile command error: %w", err) + } + if rm { + cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, rmOutCMD) + if err := utils.RunCmdViaSSHContext(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { + return fmt.Errorf("remote: rm ./out command error: %w", err) + } + } + cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, compileCMD) + if err := utils.RunCmdViaSSHContextNoRetry(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { + return fmt.Errorf("remote: compile command error: %w", err) + } + // has been built already, pitiful if canceled, so continue copying + for _, f := range imgList { + imgName := filepath.Base(f) + if err := utils.TransFileViaSSH(utils.Download, server.Addr, server.User, server.Passwd, + fmt.Sprintf("%s/%s", server.WorkSpace, f), filepath.Join(m.Workspace, pkg, imgName)); err != nil { + return fmt.Errorf("download file %s error: %w", f, err) + } + } + return nil +} diff --git a/tools/fotff/pkg/dayu200/dayu200.go b/tools/fotff/pkg/dayu200/dayu200.go new file mode 100644 index 0000000000000000000000000000000000000000..17ae273e33d466e4b16b8b18db7b49921eb26be8 --- /dev/null +++ b/tools/fotff/pkg/dayu200/dayu200.go @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dayu200 + +import ( + "code.cloudfoundry.org/archiver/extractor" + "context" + "fmt" + "fotff/pkg" + "fotff/res" + "fotff/utils" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +type Manager struct { + ArchiveDir string `key:"archive_dir" default:"."` + FromCI string `key:"download_from_ci" default:"false"` + Workspace string `key:"workspace" default:"."` + Branch string `key:"branch" default:"master"` + FlashTool string `key:"flash_tool" default:"python"` + LocationIDList string `key:"location_id_list"` + locations map[string]string + fromCI bool +} + +func NewManager() pkg.Manager { + var ret Manager + utils.ParseFromConfigFile("dayu200", &ret) + var err error + if ret.fromCI, err = strconv.ParseBool(ret.FromCI); err != nil { + logrus.Panicf("can not parse 'download_from_ci', please check") + } + devs := res.DeviceList() + locs := strings.Split(ret.LocationIDList, ",") + if len(devs) != len(locs) { + logrus.Panicf("location_id_list and devices mismatch") + } + ret.locations = map[string]string{} + for i, loc := range locs { + ret.locations[devs[i]] = loc + } + go ret.cleanupOutdated() + return &ret +} + +func (m *Manager) cleanupOutdated() { + t := time.NewTicker(24 * time.Hour) + for { + <-t.C + es, err := os.ReadDir(m.Workspace) + if err != nil { + logrus.Errorf("can not read %s: %v", m.Workspace, err) + continue + } + for _, e := range es { + if !e.IsDir() { + continue + } + path := filepath.Join(m.Workspace, e.Name()) + info, err := e.Info() + if err != nil { + logrus.Errorf("can not read %s info: %v", path, err) + continue + } + if time.Now().Sub(info.ModTime()) > 7*24*time.Hour { + logrus.Warnf("%s outdated, cleanning up its contents...", path) + m.cleanupPkgFiles(path) + } + } + } +} + +func (m *Manager) cleanupPkgFiles(path string) { + es, err := os.ReadDir(path) + if err != nil { + logrus.Errorf("can not read %s: %v", path, err) + return + } + for _, e := range es { + if e.Name() == "manifest_tag.xml" || e.Name() == "__last_issue__" { + continue + } + if err := os.RemoveAll(filepath.Join(path, e.Name())); err != nil { + logrus.Errorf("remove %s fail: %v", filepath.Join(path, e.Name()), err) + } + } +} + +// Flash function implements pkg.Manager. Flash images in the 'pkg' directory to the given device. +// If not all necessary images are available in the 'pkg' directory, will build them. +func (m *Manager) Flash(device string, pkg string, ctx context.Context) error { + logrus.Infof("now flash %s", pkg) + if !m.pkgAvailable(pkg) { + logrus.Infof("%s is not available", pkg) + if err := m.build(pkg, false, ctx); err != nil { + logrus.Errorf("build pkg %s err: %v", pkg, err) + logrus.Infof("build pkg %s again...", pkg) + if err = m.build(pkg, true, ctx); err != nil { + logrus.Errorf("build pkg %s err: %v", pkg, err) + return err + } + } + } + logrus.Infof("%s is available now, start to flash it", pkg) + return m.flashDevice(device, pkg, ctx) +} + +func (m *Manager) Steps(from, to string) (pkgs []string, err error) { + if from == to { + return nil, fmt.Errorf("steps err: 'from' %s and 'to' %s are the same", from, to) + } + if c, found := utils.CacheGet("dayu200_steps", from+"__to__"+to); found { + logrus.Infof("steps from %s to %s are cached", from, to) + logrus.Infof("steps: %v", c.([]string)) + return c.([]string), nil + } + if pkgs, err = m.stepsFromGitee(from, to); err != nil { + logrus.Errorf("failed to gen steps from gitee, err: %v", err) + logrus.Warnf("fallback to getting steps from CI...") + if pkgs, err = m.stepsFromCI(from, to); err != nil { + return pkgs, err + } + return pkgs, nil + } + utils.CacheSet("dayu200_steps", from+"__to__"+to, pkgs) + return pkgs, nil +} + +func (m *Manager) LastIssue(pkg string) (string, error) { + data, err := os.ReadFile(filepath.Join(m.Workspace, pkg, "__last_issue__")) + return string(data), err +} + +func (m *Manager) GetNewer(cur string) (string, error) { + var newFile string + if m.fromCI { + newFile = m.getNewerFromCI(cur + ".tar.gz") + } else { + newFile = pkg.GetNewerFileFromDir(m.ArchiveDir, cur+".tar.gz", func(files []os.DirEntry, i, j int) bool { + ti, _ := getPackageTime(files[i].Name()) + tj, _ := getPackageTime(files[j].Name()) + return ti.Before(tj) + }) + } + ex := extractor.NewTgz() + dirName := newFile + for filepath.Ext(dirName) != "" { + dirName = strings.TrimSuffix(dirName, filepath.Ext(dirName)) + } + dir := filepath.Join(m.Workspace, dirName) + if _, err := os.Stat(dir); err == nil { + return dirName, nil + } + logrus.Infof("extracting %s to %s...", filepath.Join(m.ArchiveDir, newFile), dir) + if err := ex.Extract(filepath.Join(m.ArchiveDir, newFile), dir); err != nil { + return dirName, err + } + return dirName, nil +} + +func (m *Manager) PkgDir(pkg string) string { + return filepath.Join(m.Workspace, pkg) +} diff --git a/tools/fotff/pkg/dayu200/flash.go b/tools/fotff/pkg/dayu200/flash.go new file mode 100644 index 0000000000000000000000000000000000000000..6ad078e6198979d6022ceeca1a42f1b70f757535 --- /dev/null +++ b/tools/fotff/pkg/dayu200/flash.go @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dayu200 + +import ( + "context" + "errors" + "fmt" + "fotff/utils" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +var partList = []string{"boot_linux", "system", "vendor", "userdata", "resource", "ramdisk", "chipset", "sys-prod", "chip-prod", "updater"} + +// All timeouts are calculated on normal cases, we do not certain that timeouts are enough if some sleeps canceled. +// So simply we do not cancel any Sleep(). TODO: use utils.SleepContext() instead. +func (m *Manager) flashDevice(device string, pkg string, ctx context.Context) error { + if err := utils.TryRebootToLoader(device, ctx); err != nil { + return err + } + if err := m.flashImages(device, pkg, ctx); err != nil { + return err + } + time.Sleep(20 * time.Second) // usually, it takes about 20s to reboot into OpenHarmony + if connected := utils.WaitHDC(device, ctx); !connected { + logrus.Errorf("flash device %s done, but boot unnormally, hdc connection fail", device) + return fmt.Errorf("flash device %s done, but boot unnormally, hdc connection fail", device) + } + time.Sleep(10 * time.Second) // wait 10s more to ensure system has been started completely + logrus.Infof("flash device %s successfully", device) + return nil +} + +func (m *Manager) flashImages(device string, pkg string, ctx context.Context) error { + logrus.Infof("calling flash tool to flash %s into %s...", pkg, device) + locationID := m.locations[device] + if locationID == "" { + data, _ := utils.ExecCombinedOutputContext(ctx, m.FlashTool, "LD") + locationID = strings.TrimPrefix(regexp.MustCompile(`LocationID=\d+`).FindString(string(data)), "LocationID=") + if locationID == "" { + time.Sleep(5 * time.Second) + data, _ := utils.ExecCombinedOutputContext(ctx, m.FlashTool, "LD") + locationID = strings.TrimPrefix(regexp.MustCompile(`LocationID=\d+`).FindString(string(data)), "LocationID=") + } + } + logrus.Infof("locationID of %s is [%s]", device, locationID) + if err := utils.ExecContext(ctx, m.FlashTool, "-s", locationID, "UL", filepath.Join(m.Workspace, pkg, "MiniLoaderAll.bin"), "-noreset"); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + logrus.Errorf("flash MiniLoaderAll.bin fail: %v", err) + time.Sleep(5 * time.Second) + if err := utils.ExecContext(ctx, m.FlashTool, "-s", locationID, "UL", filepath.Join(m.Workspace, pkg, "MiniLoaderAll.bin"), "-noreset"); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + logrus.Errorf("flash MiniLoaderAll.bin fail: %v", err) + return err + } + } + time.Sleep(3 * time.Second) + if err := utils.ExecContext(ctx, m.FlashTool, "-s", locationID, "DI", "-p", filepath.Join(m.Workspace, pkg, "parameter.txt")); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + logrus.Errorf("flash parameter.txt fail: %v", err) + return err + } + time.Sleep(5 * time.Second) + if err := utils.ExecContext(ctx, m.FlashTool, "-s", locationID, "DI", "-uboot", filepath.Join(m.Workspace, pkg, "uboot.img"), filepath.Join(m.Workspace, pkg, "parameter.txt")); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + logrus.Errorf("flash device fail: %v", err) + return err + } + time.Sleep(5 * time.Second) + for _, part := range partList { + if _, err := os.Stat(filepath.Join(m.Workspace, pkg, part+".img")); err != nil { + if os.IsNotExist(err) { + logrus.Infof("part %s.img not exist, ignored", part) + continue + } + return err + } + if err := utils.ExecContext(ctx, m.FlashTool, "-s", locationID, "DI", "-"+part, filepath.Join(m.Workspace, pkg, part+".img"), filepath.Join(m.Workspace, pkg, "parameter.txt")); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + logrus.Errorf("flash device fail: %v", err) + logrus.Warnf("try again...") + if err := utils.ExecContext(ctx, m.FlashTool, "-s", locationID, "DI", "-"+part, filepath.Join(m.Workspace, pkg, part+".img"), filepath.Join(m.Workspace, pkg, "parameter.txt")); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + logrus.Errorf("flash device fail: %v", err) + return err + } + } + time.Sleep(3 * time.Second) + } + time.Sleep(5 * time.Second) // sleep a while for writing + if err := utils.ExecContext(ctx, m.FlashTool, "-s", locationID, "RD"); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + return fmt.Errorf("reboot device fail: %v", err) + } + return nil +} diff --git a/tools/fotff/pkg/dayu200/get_newer_ci.go b/tools/fotff/pkg/dayu200/get_newer_ci.go new file mode 100644 index 0000000000000000000000000000000000000000..b845e2db60f1ae51bf97881fb165de2c27a4140d --- /dev/null +++ b/tools/fotff/pkg/dayu200/get_newer_ci.go @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dayu200 + +import ( + "encoding/json" + "fotff/utils" + "github.com/sirupsen/logrus" + "io" + "net/http" + "os" + "path/filepath" + "time" +) + +type DailyBuildsQueryParam struct { + ProjectName string `json:"projectName"` + Branch string `json:"branch"` + Component string `json:"component"` + BuildStatus string `json:"buildStatus"` + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` + PageNum int `json:"pageNum"` + PageSize int `json:"pageSize"` +} + +type DailyBuildsResp struct { + Result struct { + DailyBuildVos []*DailyBuild `json:"dailyBuildVos"` + Total int `json:"total"` + } `json:"result"` +} + +type DailyBuild struct { + Id string `json:"id"` + ImgObsPath string `json:"imgObsPath"` +} + +func (m *Manager) getNewerFromCI(cur string) string { + for { + file := func() string { + var q = DailyBuildsQueryParam{ + ProjectName: "openharmony", + Branch: m.Branch, + Component: "dayu200", + BuildStatus: "success", + PageNum: 1, + PageSize: 1, + } + data, err := json.Marshal(q) + if err != nil { + logrus.Errorf("can not marshal query param: %v", err) + return "" + } + resp, err := utils.DoSimpleHttpReq(http.MethodPost, "http://ci.openharmony.cn/api/ci-backend/ci-portal/v1/dailybuilds", data, map[string]string{"Content-Type": "application/json;charset=UTF-8"}) + if err != nil { + logrus.Errorf("can not query builds: %v", err) + return "" + } + var dailyBuildsResp DailyBuildsResp + if err := json.Unmarshal(resp, &dailyBuildsResp); err != nil { + logrus.Errorf("can not unmarshal resp [%s]: %v", string(resp), err) + return "" + } + if len(dailyBuildsResp.Result.DailyBuildVos) != 0 { + url := dailyBuildsResp.Result.DailyBuildVos[0].ImgObsPath + if filepath.Base(url) != cur { + logrus.Infof("new package found, name: %s", filepath.Base(url)) + file, err := m.downloadToWorkspace(url) + if err != nil { + logrus.Errorf("can not download package %s: %v", url, err) + return "" + } + return file + } + } + return "" + }() + if file != "" { + return file + } + time.Sleep(10 * time.Minute) + } +} + +func (m *Manager) downloadToWorkspace(url string) (string, error) { + logrus.Infof("downloading %s", url) + resp, err := utils.DoSimpleHttpReqRaw(http.MethodGet, url, nil, nil) + if err != nil { + return "", err + } + defer resp.Body.Close() + if err := os.MkdirAll(m.ArchiveDir, 0750); err != nil { + return "", err + } + f, err := os.Create(filepath.Join(m.ArchiveDir, filepath.Base(url))) + if err != nil { + return "", err + } + defer f.Close() + if _, err := io.CopyBuffer(f, resp.Body, make([]byte, 16*1024*1024)); err != nil { + return "", err + } + logrus.Infof("%s downloaded successfully", url) + return filepath.Base(url), nil +} diff --git a/tools/fotff/pkg/dayu200/steps_ci.go b/tools/fotff/pkg/dayu200/steps_ci.go new file mode 100644 index 0000000000000000000000000000000000000000..577f9f2691f88431c18ab34a78d7ae52038a63ef --- /dev/null +++ b/tools/fotff/pkg/dayu200/steps_ci.go @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dayu200 + +import ( + "encoding/json" + "fmt" + "fotff/utils" + "github.com/sirupsen/logrus" + "net/http" + "os" + "path/filepath" + "sort" + "time" +) + +type TagQueryParam struct { + ProjectName string `json:"projectName"` + Branch string `json:"branch"` + ManifestFile string `json:"manifestFile"` + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` + PageNum int `json:"pageNum"` + PageSize int `json:"pageSize"` +} + +type TagResp struct { + Result struct { + TagList []*Tag `json:"tagList"` + Total int `json:"total"` + } `json:"result"` +} + +type Tag struct { + Id string `json:"id"` + Issue string `json:"issue"` + PrList []string `json:"prList"` + TagFileURL string `json:"tagFileUrl"` + Timestamp string `json:"timestamp"` +} + +func (m *Manager) stepsFromCI(from, to string) (pkgs []string, err error) { + startTime, err := getPackageTime(from) + if err != nil { + return nil, err + } + endTime, err := getPackageTime(to) + if err != nil { + return nil, err + } + return m.getAllStepsFromTags(startTime, endTime) +} + +func (m *Manager) getAllStepsFromTags(from, to time.Time) (pkgs []string, err error) { + tags, err := m.getAllTags(from, to) + if err != nil { + return nil, err + } + sort.Slice(tags, func(i, j int) bool { + return tags[i].Timestamp < tags[j].Timestamp + }) + for _, tag := range tags { + pkg, err := m.genTagPackage(tag) + if err != nil { + return nil, err + } + pkgs = append(pkgs, pkg) + } + return pkgs, nil +} + +func (m *Manager) getAllTags(from, to time.Time) (ret []*Tag, err error) { + var deDup = make(map[string]*Tag) + var pageNum = 1 + for { + var q = TagQueryParam{ + ProjectName: "openharmony", + Branch: m.Branch, + ManifestFile: "default.xml", + StartTime: from.Local().Format("2006-01-02"), + EndTime: to.Local().Format("2006-01-02"), + PageNum: pageNum, + PageSize: 10000, + } + data, err := json.Marshal(q) + if err != nil { + return nil, err + } + resp, err := utils.DoSimpleHttpReq(http.MethodPost, "http://ci.openharmony.cn/api/ci-backend/ci-portal/v1/build/tag", data, map[string]string{"Content-Type": "application/json;charset=UTF-8"}) + if err != nil { + return nil, err + } + var tagResp TagResp + if err := json.Unmarshal(resp, &tagResp); err != nil { + return nil, err + } + for _, tag := range tagResp.Result.TagList { + if _, ok := deDup[tag.Id]; ok { + continue + } + deDup[tag.Id] = tag + date, err := time.ParseInLocation("2006-01-02 15:04:05", tag.Timestamp, time.Local) + if err != nil { + return nil, err + } + if date.After(from) && date.Before(to) { + ret = append(ret, tag) + } + } + if len(deDup) == tagResp.Result.Total { + break + } + pageNum++ + } + return ret, nil +} + +func (m *Manager) genTagPackage(tag *Tag) (pkg string, err error) { + defer func() { + logrus.Infof("package dir %s for tag %v generated", pkg, tag.TagFileURL) + }() + if err := os.MkdirAll(filepath.Join(m.Workspace, tag.Id), 0750); err != nil { + return "", err + } + var issues []string + if len(tag.Issue) == 0 { + issues = tag.PrList + } else { + issues = []string{tag.Issue} + } + if err := os.WriteFile(filepath.Join(m.Workspace, tag.Id, "__last_issue__"), []byte(fmt.Sprintf("%v", issues)), 0640); err != nil { + return "", err + } + resp, err := utils.DoSimpleHttpReq(http.MethodGet, tag.TagFileURL, nil, nil) + if err != nil { + return "", err + } + err = os.WriteFile(filepath.Join(m.Workspace, tag.Id, "manifest_tag.xml"), resp, 0640) + if err != nil { + return "", err + } + return tag.Id, nil +} diff --git a/tools/fotff/pkg/dayu200/steps_gitee.go b/tools/fotff/pkg/dayu200/steps_gitee.go new file mode 100644 index 0000000000000000000000000000000000000000..543c82ad1a338d13d422024d81b5180c3d6faaae --- /dev/null +++ b/tools/fotff/pkg/dayu200/steps_gitee.go @@ -0,0 +1,365 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dayu200 + +import ( + "bufio" + "bytes" + "encoding/xml" + "fmt" + "fotff/vcs" + "fotff/vcs/gitee" + "github.com/huandu/go-clone" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +type IssueInfo struct { + visited bool + RelatedIssues []string + MRs []*gitee.Commit + StructCTime string + StructureUpdates []*vcs.ProjectUpdate +} + +type Step struct { + IssueURLs []string + MRs []*gitee.Commit + StructCTime string + StructureUpdates []*vcs.ProjectUpdate +} + +func (m *Manager) stepsFromGitee(from, to string) (pkgs []string, err error) { + updates, err := m.getRepoUpdates(from, to) + if err != nil { + return nil, err + } + startTime, err := getPackageTime(from) + if err != nil { + return nil, err + } + endTime, err := getPackageTime(to) + if err != nil { + return nil, err + } + logrus.Infof("find %d repo updates from %s to %s", len(updates), from, to) + steps, err := getAllStepsFromGitee(startTime, endTime, m.Branch, updates) + if err != nil { + return nil, err + } + logrus.Infof("find total %d steps from %s to %s", len(steps), from, to) + baseManifest, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml")) + if err != nil { + return nil, err + } + for _, step := range steps { + var newPkg string + if newPkg, baseManifest, err = m.genStepPackage(baseManifest, step); err != nil { + return nil, err + } + pkgs = append(pkgs, newPkg) + } + return pkgs, nil +} + +func (m *Manager) getRepoUpdates(from, to string) (updates []vcs.ProjectUpdate, err error) { + m1, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml")) + if err != nil { + return nil, err + } + m2, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, to, "manifest_tag.xml")) + if err != nil { + return nil, err + } + return vcs.GetRepoUpdates(m1, m2) +} + +func getAllStepsFromGitee(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (ret []Step, err error) { + allMRs, err := getAllMRs(startTime, endTime, branch, updates) + if err != nil { + return nil, err + } + issueInfos, err := combineMRsToIssue(allMRs, branch) + if err != nil { + return nil, err + } + return combineIssuesToStep(issueInfos) +} + +func getAllMRs(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (allMRs []*gitee.Commit, err error) { + var once sync.Once + for _, update := range updates { + var prs []*gitee.Commit + if update.P1.StructureDiff(update.P2) { + once.Do(func() { + prs, err = gitee.GetBetweenTimeMRs("openharmony", "manifest", branch, startTime, endTime) + }) + if update.P1 != nil { + var p1 []*gitee.Commit + p1, err = gitee.GetBetweenTimeMRs("openharmony", update.P1.Name, branch, startTime, endTime) + prs = append(prs, p1...) + } + if update.P2 != nil { + var p2 []*gitee.Commit + p2, err = gitee.GetBetweenTimeMRs("openharmony", update.P2.Name, branch, startTime, endTime) + prs = append(prs, p2...) + } + } else { + prs, err = gitee.GetBetweenMRs(gitee.CompareParam{ + Head: update.P2.Revision, + Base: update.P1.Revision, + Owner: "openharmony", + Repo: update.P2.Name, + }) + } + if err != nil { + return nil, err + } + allMRs = append(allMRs, prs...) + } + logrus.Infof("find total %d merge request commits of all repo updates", len(allMRs)) + return +} + +func combineMRsToIssue(allMRs []*gitee.Commit, branch string) (map[string]*IssueInfo, error) { + ret := make(map[string]*IssueInfo) + for _, mr := range allMRs { + num, err := strconv.Atoi(strings.Trim(regexp.MustCompile(`!\d+ `).FindString(mr.Commit.Message), "! ")) + if err != nil { + return nil, fmt.Errorf("parse MR message for %s fail: %s", mr.URL, err) + } + issues, err := gitee.GetMRIssueURL(mr.Owner, mr.Repo, num) + if err != nil { + return nil, err + } + if len(issues) == 0 { + issues = []string{mr.URL} + } + var scs []*vcs.ProjectUpdate + var scTime string + if mr.Owner == "openharmony" && mr.Repo == "manifest" { + if scTime, scs, err = parseStructureUpdates(mr, branch); err != nil { + return nil, err + } + } + for i, issue := range issues { + if _, ok := ret[issue]; !ok { + ret[issue] = &IssueInfo{ + MRs: []*gitee.Commit{mr}, + RelatedIssues: append(issues[:i], issues[i+1:]...), + StructCTime: scTime, + StructureUpdates: scs, + } + } else { + ret[issue] = &IssueInfo{ + MRs: append(ret[issue].MRs, mr), + RelatedIssues: append(ret[issue].RelatedIssues, append(issues[:i], issues[i+1:]...)...), + StructCTime: scTime, + StructureUpdates: append(ret[issue].StructureUpdates, scs...), + } + } + } + } + logrus.Infof("find total %d issues of all repo updates", len(ret)) + return ret, nil +} + +func combineOtherRelatedIssue(parent, self *IssueInfo, all map[string]*IssueInfo) { + if self.visited { + return + } + self.visited = true + for _, other := range self.RelatedIssues { + if son, ok := all[other]; ok { + combineOtherRelatedIssue(self, son, all) + delete(all, other) + } + } + parent.RelatedIssues = deDupIssues(append(parent.RelatedIssues, self.RelatedIssues...)) + parent.MRs = deDupMRs(append(parent.MRs, self.MRs...)) + parent.StructureUpdates = deDupProjectUpdates(append(parent.StructureUpdates, self.StructureUpdates...)) + if len(parent.StructCTime) != 0 && parent.StructCTime < self.StructCTime { + parent.StructCTime = self.StructCTime + } +} + +func deDupProjectUpdates(us []*vcs.ProjectUpdate) (retMRs []*vcs.ProjectUpdate) { + dupIndexes := make([]bool, len(us)) + for i := range us { + for j := i + 1; j < len(us); j++ { + if us[j].P1 == us[i].P1 && us[j].P2 == us[i].P2 { + dupIndexes[j] = true + } + } + } + for i, dup := range dupIndexes { + if dup { + continue + } + retMRs = append(retMRs, us[i]) + } + return +} + +func deDupMRs(mrs []*gitee.Commit) (retMRs []*gitee.Commit) { + tmp := make(map[string]*gitee.Commit) + for _, m := range mrs { + tmp[m.SHA] = m + } + for _, m := range tmp { + retMRs = append(retMRs, m) + } + return +} + +func deDupIssues(issues []string) (retIssues []string) { + tmp := make(map[string]string) + for _, i := range issues { + tmp[i] = i + } + for _, i := range tmp { + retIssues = append(retIssues, i) + } + return +} + +// parseStructureUpdates get changed XMLs and parse it to recognize repo structure changes. +// Since we do not care which revision a repo was, P1 is not welly handled, just assign it not nil for performance. +func parseStructureUpdates(commit *gitee.Commit, branch string) (string, []*vcs.ProjectUpdate, error) { + tmp := make(map[string]vcs.ProjectUpdate) + if len(commit.Files) == 0 { + // commit that queried from MR req does not contain file details, should fetch again + var err error + if commit, err = gitee.GetCommit(commit.Owner, commit.Repo, commit.SHA); err != nil { + return "", nil, err + } + } + for _, f := range commit.Files { + if filepath.Ext(f.Filename) != ".xml" { + continue + } + if err := parseFilePatch(f.Patch, tmp); err != nil { + return "", nil, err + } + } + var ret []*vcs.ProjectUpdate + for _, pu := range tmp { + projectUpdateCopy := pu + ret = append(ret, &projectUpdateCopy) + } + for _, pu := range ret { + if pu.P1 == nil && pu.P2 != nil { + lastCommit, err := gitee.GetLatestMRBefore("openharmony", pu.P2.Name, branch, commit.Commit.Committer.Date) + if err != nil { + return "", nil, err + } + pu.P2.Revision = lastCommit.SHA + } + } + return commit.Commit.Committer.Date, ret, nil +} + +func parseFilePatch(str string, m map[string]vcs.ProjectUpdate) error { + sc := bufio.NewScanner(bytes.NewBuffer([]byte(str))) + for sc.Scan() { + line := sc.Text() + var p vcs.Project + if strings.HasPrefix(line, "-") { + if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil { + m[p.Name] = vcs.ProjectUpdate{P1: &p, P2: m[p.Name].P2} + } + } else if strings.HasPrefix(line, "+") { + if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil { + m[p.Name] = vcs.ProjectUpdate{P1: m[p.Name].P1, P2: &p} + } + } + } + return nil +} + +func combineIssuesToStep(issueInfos map[string]*IssueInfo) (ret []Step, err error) { + for _, info := range issueInfos { + combineOtherRelatedIssue(info, info, issueInfos) + } + for issue, infos := range issueInfos { + sort.Slice(infos.MRs, func(i, j int) bool { + // move the latest MR to the first place, use its merged_time to represent the update time of the issue + return infos.MRs[i].Commit.Committer.Date > infos.MRs[j].Commit.Committer.Date + }) + ret = append(ret, Step{ + IssueURLs: append(infos.RelatedIssues, issue), + MRs: infos.MRs, + StructCTime: infos.StructCTime, + StructureUpdates: infos.StructureUpdates}) + } + sort.Slice(ret, func(i, j int) bool { + ti, tj := ret[i].MRs[0].Commit.Committer.Date, ret[j].MRs[0].Commit.Committer.Date + if len(ret[i].StructCTime) != 0 { + ti = ret[i].StructCTime + } + if len(ret[j].StructCTime) != 0 { + ti = ret[j].StructCTime + } + return ti < tj + }) + logrus.Infof("find total %d steps of all issues", len(ret)) + return +} + +var simpleRegTimeInPkgName = regexp.MustCompile(`\d{8}_\d{6}`) + +func getPackageTime(pkg string) (time.Time, error) { + return time.ParseInLocation(`20060102_150405`, simpleRegTimeInPkgName.FindString(pkg), time.Local) +} + +func (m *Manager) genStepPackage(base *vcs.Manifest, step Step) (newPkg string, newManifest *vcs.Manifest, err error) { + defer func() { + logrus.Infof("package dir %s for step %v generated", newPkg, step.IssueURLs) + }() + newManifest = clone.Clone(base).(*vcs.Manifest) + for _, u := range step.StructureUpdates { + if u.P2 != nil { + newManifest.UpdateManifestProject(u.P2.Name, u.P2.Path, u.P2.Remote, u.P2.Revision, true) + } else if u.P1 != nil { + newManifest.RemoveManifestProject(u.P1.Name) + } + } + for _, mr := range step.MRs { + newManifest.UpdateManifestProject(mr.Repo, "", "", mr.SHA, false) + } + md5sum, err := newManifest.Standardize() + if err != nil { + return "", nil, err + } + if err := os.MkdirAll(filepath.Join(m.Workspace, md5sum), 0750); err != nil { + return "", nil, err + } + if err := os.WriteFile(filepath.Join(m.Workspace, md5sum, "__last_issue__"), []byte(fmt.Sprintf("%v", step.IssueURLs)), 0640); err != nil { + return "", nil, err + } + err = newManifest.WriteFile(filepath.Join(m.Workspace, md5sum, "manifest_tag.xml")) + if err != nil { + return "", nil, err + } + return md5sum, newManifest, nil +} diff --git a/tools/fotff/pkg/dayu200/steps_gitee_test.go b/tools/fotff/pkg/dayu200/steps_gitee_test.go new file mode 100644 index 0000000000000000000000000000000000000000..f7b86f18b2861da2f4646a70c85eea93ff49aeca --- /dev/null +++ b/tools/fotff/pkg/dayu200/steps_gitee_test.go @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dayu200 + +import ( + "fotff/vcs" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestMain(m *testing.M) { + defer os.RemoveAll(".fotff") + defer os.RemoveAll("logs") + m.Run() +} + +func TestManager_Steps(t *testing.T) { + m := &Manager{Workspace: "./testdata", Branch: "master"} + defer func() { + entries, _ := os.ReadDir(m.Workspace) + for _, e := range entries { + if strings.HasPrefix(e.Name(), "version") { + continue + } + os.RemoveAll(filepath.Join(m.Workspace, e.Name())) + } + }() + tests := []struct { + name string + from, to string + stepsNum int + }{ + { + name: "15 MR of 15 steps in 12 repo, with 1 path change", + from: "version-Daily_Version-dayu200-20221201_080109-dayu200", + to: "version-Daily_Version-dayu200-20221201_100141-dayu200", + stepsNum: 15, + }, + { + name: "27 MR of 25 steps in 21 repo, with 1 repo add", + from: "version-Daily_Version-dayu200-20221213_110027-dayu200", + to: "version-Daily_Version-dayu200-20221213_140150-dayu200", + stepsNum: 25, + }, + { + name: "15 MR of 14 steps in 14 repo, no structure change", + from: "version-Daily_Version-dayu200-20221214_100124-dayu200", + to: "version-Daily_Version-dayu200-20221214_110125-dayu200", + stepsNum: 14, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ret, err := m.Steps(tt.from, tt.to) + if err != nil { + t.Fatalf("err: expcect: , actual: %v", err) + } + if len(ret) != tt.stepsNum { + t.Fatalf("steps num: expcect: %d, actual: %v", tt.stepsNum, len(ret)) + } + if tt.stepsNum == 0 { + return + } + mLast, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, ret[len(ret)-1], "manifest_tag.xml")) + if err != nil { + t.Fatalf("err: expcect: , actual: %v", err) + } + mLastMD5, err := mLast.Standardize() + if err != nil { + t.Fatalf("err: expcect: , actual: %v", err) + } + expected, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, tt.to, "manifest_tag.xml")) + if err != nil { + t.Fatalf("err: expcect: , actual: %v", err) + } + expectedMD5, err := expected.Standardize() + if err != nil { + t.Fatalf("err: expcect: , actual: %v", err) + } + if mLastMD5 != expectedMD5 { + t.Errorf("steps result: expect: %s, actual: %s", expectedMD5, mLastMD5) + } + }) + } +} diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_080109-dayu200/manifest_tag.xml b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_080109-dayu200/manifest_tag.xml new file mode 100644 index 0000000000000000000000000000000000000000..d6cddc4f84e1c97f3281172e58b8b0e2667b0774 --- /dev/null +++ b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_080109-dayu200/manifest_tag.xml @@ -0,0 +1,411 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_100141-dayu200/manifest_tag.xml b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_100141-dayu200/manifest_tag.xml new file mode 100644 index 0000000000000000000000000000000000000000..700fe6bff5631e2844556d8538f25f7a7e8cf82a --- /dev/null +++ b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_100141-dayu200/manifest_tag.xml @@ -0,0 +1,411 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_110027-dayu200/manifest_tag.xml b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_110027-dayu200/manifest_tag.xml new file mode 100644 index 0000000000000000000000000000000000000000..51b92b7533cd97cf832bd6b324678c44fecf4462 --- /dev/null +++ b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_110027-dayu200/manifest_tag.xml @@ -0,0 +1,411 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_140150-dayu200/manifest_tag.xml b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_140150-dayu200/manifest_tag.xml new file mode 100644 index 0000000000000000000000000000000000000000..c1b47deb12363c282b03258f1da25c5c15ba806b --- /dev/null +++ b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_140150-dayu200/manifest_tag.xml @@ -0,0 +1,412 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_100124-dayu200/manifest_tag.xml b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_100124-dayu200/manifest_tag.xml new file mode 100644 index 0000000000000000000000000000000000000000..b1f452ec78dd5f9eeed10604e06153de8916964e --- /dev/null +++ b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_100124-dayu200/manifest_tag.xml @@ -0,0 +1,412 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_110125-dayu200/manifest_tag.xml b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_110125-dayu200/manifest_tag.xml new file mode 100644 index 0000000000000000000000000000000000000000..580e14f1e3524bec2adecb05842c409773f6cb0a --- /dev/null +++ b/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_110125-dayu200/manifest_tag.xml @@ -0,0 +1,412 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/fotff/pkg/mock/mock.go b/tools/fotff/pkg/mock/mock.go new file mode 100644 index 0000000000000000000000000000000000000000..be38bf5850b127ea601fd66e2afd099ef929f405 --- /dev/null +++ b/tools/fotff/pkg/mock/mock.go @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mock + +import ( + "context" + "fmt" + "fotff/pkg" + "github.com/sirupsen/logrus" + "time" +) + +type Manager struct { + pkgCount int +} + +func NewManager() pkg.Manager { + return &Manager{} +} + +func (m *Manager) LastIssue(pkg string) (string, error) { + ret := fmt.Sprintf("https://testserver.com/issues/%s", pkg) + logrus.Infof("LastIssue: mock implementation returns %s", ret) + return ret, nil +} + +func (m *Manager) Steps(from, to string) ([]string, error) { + var ret = []string{"step1", "step2", "step3"} + for i := range ret { + ret[i] = fmt.Sprintf("%s-%s-%s", from, to, ret[i]) + } + logrus.Infof("Steps: mock implementation returns %v", ret) + return ret, nil +} + +func (m *Manager) GetNewer(cur string) (string, error) { + ret := fmt.Sprintf("pkg%d", m.pkgCount) + time.Sleep(time.Duration(m.pkgCount) * time.Second) + m.pkgCount++ + logrus.Infof("GetNewer: mock implementation returns %s", ret) + return ret, nil +} + +func (m *Manager) Flash(device string, pkg string, ctx context.Context) error { + time.Sleep(time.Second) + logrus.Infof("Flash: flashing %s to %s, mock implementation returns OK unconditionally", pkg, device) + return nil +} + +func (m *Manager) PkgDir(pkg string) string { + return pkg +} diff --git a/tools/fotff/pkg/pkg.go b/tools/fotff/pkg/pkg.go new file mode 100644 index 0000000000000000000000000000000000000000..142ae30d73aec678e24cfd3e4efda0f96ecf4a67 --- /dev/null +++ b/tools/fotff/pkg/pkg.go @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pkg + +import ( + "context" + "github.com/sirupsen/logrus" + "os" + "sort" + "time" +) + +type NewFunc func() Manager + +type Manager interface { + // Flash download given package to the device. + Flash(device string, pkg string, ctx context.Context) error + // LastIssue returns the last issue URL related to the package. + LastIssue(pkg string) (string, error) + // Steps generates every intermediate package and returns the list sequentially. + Steps(from, to string) ([]string, error) + // GetNewer blocks the process until a newer package is found, then returns the newest one. + GetNewer(cur string) (string, error) + // PkgDir returns where pkg exists in the filesystem. + PkgDir(pkg string) string +} + +func GetNewerFileFromDir(dir string, cur string, less func(files []os.DirEntry, i, j int) bool) string { + for { + files, err := os.ReadDir(dir) + if err != nil { + logrus.Errorf("read dir %s err: %s", dir, err) + time.Sleep(10 * time.Second) + continue + } + sort.Slice(files, func(i, j int) bool { + return less(files, i, j) + }) + if len(files) != 0 { + f := files[len(files)-1] + if f.Name() != cur { + logrus.Infof("new package found, name: %s", f.Name()) + return f.Name() + } + } + time.Sleep(10 * time.Second) + } +} diff --git a/tools/fotff/rec/fotff.go b/tools/fotff/rec/fotff.go new file mode 100644 index 0000000000000000000000000000000000000000..84e108fe916b7c84d620316a5de0a2acc71abf66 --- /dev/null +++ b/tools/fotff/rec/fotff.go @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rec + +import ( + "context" + "errors" + "fmt" + "fotff/pkg" + "fotff/res" + "fotff/tester" + "fotff/utils" + "github.com/sirupsen/logrus" + "math" + "sync" +) + +type cancelCtx struct { + ctx context.Context + fn context.CancelFunc +} + +// FindOutTheFirstFail returns the first issue URL that introduce the failure. +// 'fellows' are optional, these testcases may be tested with target testcase together. +func FindOutTheFirstFail(m pkg.Manager, t tester.Tester, testCase string, successPkg string, failPkg string, fellows ...string) (string, error) { + if successPkg == "" { + return "", fmt.Errorf("can not get a success package for %s", testCase) + } + steps, err := m.Steps(successPkg, failPkg) + if err != nil { + return "", err + } + return findOutTheFirstFail(m, t, testCase, steps, fellows...) +} + +// findOutTheFirstFail is the recursive implementation to find out the first issue URL that introduce the failure. +// Arg steps' length must be grater than 1. The last step is a pre-known failure, while the rests are not tested. +// 'fellows' are optional. In the last recursive term, they have the same result as what the target testcases has. +// These fellows can be tested with target testcase together in this term to accelerate testing. +func findOutTheFirstFail(m pkg.Manager, t tester.Tester, testcase string, steps []string, fellows ...string) (string, error) { + if len(steps) == 0 { + return "", errors.New("steps are no between (success, failure], perhaps the failure is occasional") + } + logrus.Infof("now use %d-section search to find out the first fault, the length of range is %d, between [%s, %s]", res.Num()+1, len(steps), steps[0], steps[len(steps)-1]) + if len(steps) == 1 { + return m.LastIssue(steps[0]) + } + // calculate gaps between every check point of N-section search. At least 1, or will cause duplicated tests. + gapLen := float64(len(steps)-1) / float64(res.Num()+1) + if gapLen < 1 { + gapLen = 1 + } + // 'success' and 'fail' record the left/right steps indexes of the next term recursive call. + // Here defines functions and surrounding helpers to update success/fail indexes and cancel un-needed tests. + success, fail := -1, len(steps)-1 + var lock sync.Mutex + var contexts []cancelCtx + updateRange := func(pass bool, index int) { + lock.Lock() + defer lock.Unlock() + if pass && index > success { + success = index + for _, ctx := range contexts { + if ctx.ctx.Value("index").(int) < success { + ctx.fn() + } + } + } + if !pass && index < fail { + fail = index + for _, ctx := range contexts { + if ctx.ctx.Value("index").(int) > fail { + ctx.fn() + } + } + } + } + // Now, start all tests concurrently. + var wg sync.WaitGroup + start := make(chan struct{}) + for i := 1; i <= res.Num(); i++ { + // Since the last step is a pre-known failure, we start index from the tail to avoid testing the last one. + // Otherwise, if the last step is the only one we test this term, we can not narrow ranges to continue. + index := len(steps) - 1 - int(math.Round(float64(i)*gapLen)) + if index < 0 { + break + } + ctx, fn := context.WithCancel(context.WithValue(context.TODO(), "index", index)) + contexts = append(contexts, cancelCtx{ctx: ctx, fn: fn}) + wg.Add(1) + go func(index int, ctx context.Context) { + defer wg.Done() + // Start after all test goroutine's contexts are registered. + // Otherwise, contexts that not registered yet may out of controlling. + <-start + var pass bool + var err error + pass, fellows, err = flashAndTest(m, t, steps[index], testcase, ctx, fellows...) + if err != nil { + if errors.Is(err, context.Canceled) { + logrus.Warnf("abort to flash %s and test %s: %v", steps[index], testcase, err) + } else { + logrus.Errorf("flash %s and test %s fail: %v", steps[index], testcase, err) + } + return + } + updateRange(pass, index) + }(index, ctx) + } + close(start) + wg.Wait() + if fail-success == len(steps) { + return "", errors.New("all judgements failed, can not narrow ranges to continue") + } + return findOutTheFirstFail(m, t, testcase, steps[success+1:fail+1], fellows...) +} + +func flashAndTest(m pkg.Manager, t tester.Tester, pkg string, testcase string, ctx context.Context, fellows ...string) (bool, []string, error) { + var newFellows []string + if result, found := utils.CacheGet("testcase_result", testcase+"__at__"+pkg); found { + logrus.Infof("get testcase result %s from cache done, result is %s", result.(tester.Result).TestCaseName, result.(tester.Result).Status) + for _, fellow := range fellows { + if fellowResult, fellowFound := utils.CacheGet("testcase_result", fellow+"__at__"+pkg); fellowFound { + logrus.Infof("get testcase result %s from cache done, result is %s", fellowResult.(tester.Result).TestCaseName, fellowResult.(tester.Result).Status) + if fellowResult.(tester.Result).Status == result.(tester.Result).Status { + newFellows = append(newFellows, fellow) + } + } + } + return result.(tester.Result).Status == tester.ResultPass, newFellows, nil + } + var results []tester.Result + device := res.GetDevice() + defer res.ReleaseDevice(device) + if err := m.Flash(device, pkg, ctx); err != nil && !errors.Is(err, context.Canceled) { + // Sometimes we need to find out the first compilation failure. Treat it as a normal test failure to re-use this framework. + var cfg struct { + AllowBuildError string `key:"allow_build_err"` + } + utils.ParseFromConfigFile("", &cfg) + if cfg.AllowBuildError != "true" { + return false, newFellows, err + } + logrus.Warnf("can not flash %s to %s, assume it as a failure: %v", pkg, device, err) + for _, cases := range append(fellows, testcase) { + results = append(results, tester.Result{TestCaseName: cases, Status: tester.ResultFail}) + } + } else { + if err = t.Prepare(m.PkgDir(pkg), device, ctx); err != nil { + return false, newFellows, err + } + results, err = t.DoTestCases(device, append(fellows, testcase), ctx) + if err != nil { + return false, newFellows, err + } + } + var testcaseStatus tester.ResultStatus + for _, result := range results { + logrus.Infof("do testcase %s at %s done, result is %s", result.TestCaseName, device, result.Status) + if result.TestCaseName == testcase { + testcaseStatus = result.Status + } + utils.CacheSet("testcase_result", result.TestCaseName+"__at__"+pkg, result) + } + for _, result := range results { + if result.TestCaseName != testcase && result.Status == testcaseStatus { + newFellows = append(newFellows, result.TestCaseName) + } + } + return testcaseStatus == tester.ResultPass, newFellows, nil +} diff --git a/tools/fotff/rec/fotff_test.go b/tools/fotff/rec/fotff_test.go new file mode 100644 index 0000000000000000000000000000000000000000..1033a0c03c6df750aa3b927d039e76bd8b3dbb8e --- /dev/null +++ b/tools/fotff/rec/fotff_test.go @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rec + +import ( + "context" + "crypto/md5" + "fmt" + "fotff/res" + "fotff/tester" + "github.com/sirupsen/logrus" + "math/rand" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +type FotffMocker struct { + FirstFail int + steps []string + lock sync.Mutex + runningPkg map[string]string +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func TestMain(m *testing.M) { + defer os.RemoveAll(".fotff") + defer os.RemoveAll("logs") + m.Run() +} + +func NewFotffMocker(stepsNum int, firstFail int) *FotffMocker { + randomPrefix := fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%d", rand.Int()))))[:4] + steps := make([]string, stepsNum) + for i := 1; i <= stepsNum; i++ { + steps[i-1] = fmt.Sprintf("%s_%s", randomPrefix, strconv.Itoa(i)) + } + return &FotffMocker{ + FirstFail: firstFail, + steps: steps, + runningPkg: map[string]string{}, + } +} + +func (f *FotffMocker) TaskName() string { + return "mocker" +} + +func (f *FotffMocker) Prepare(pkgDir string, device string, ctx context.Context) error { + return nil +} + +func (f *FotffMocker) DoTestTask(device string, ctx context.Context) ([]tester.Result, error) { + time.Sleep(time.Duration(rand.Intn(1)) * time.Millisecond) + select { + case <-ctx.Done(): + return nil, context.Canceled + default: + } + return []tester.Result{{TestCaseName: f.TestCaseName(), Status: tester.ResultFail}}, nil +} + +func (f *FotffMocker) DoTestCase(device string, testcase string, ctx context.Context) (tester.Result, error) { + time.Sleep(time.Duration(rand.Intn(1)) * time.Millisecond) + select { + case <-ctx.Done(): + return tester.Result{}, context.Canceled + default: + } + f.lock.Lock() + _, pkgPrefix, _ := strings.Cut(f.runningPkg[device], "_") + pkgOrder, _ := strconv.Atoi(pkgPrefix) + f.lock.Unlock() + if pkgOrder >= f.FirstFail { + logrus.Infof("mock: test %s at %s done, result is %s", testcase, device, tester.ResultFail) + return tester.Result{TestCaseName: testcase, Status: tester.ResultFail}, nil + } + logrus.Infof("mock: test %s at %s done, result is %s", testcase, device, tester.ResultPass) + return tester.Result{TestCaseName: testcase, Status: tester.ResultPass}, nil +} + +func (f *FotffMocker) DoTestCases(device string, testcases []string, ctx context.Context) ([]tester.Result, error) { + var ret []tester.Result + for _, testcase := range testcases { + r, err := f.DoTestCase(device, testcase, ctx) + if err != nil { + return nil, err + } + ret = append(ret, r) + } + return ret, nil +} + +func (f *FotffMocker) Flash(device string, pkg string, ctx context.Context) error { + time.Sleep(time.Duration(rand.Intn(1)) * time.Millisecond) + select { + case <-ctx.Done(): + return context.Canceled + default: + } + f.lock.Lock() + f.runningPkg[device] = pkg + logrus.Infof("mock: flash %s to %s done", pkg, device) + f.lock.Unlock() + return nil +} + +func (f *FotffMocker) LastIssue(pkg string) (string, error) { + return "issue" + pkg, nil +} + +func (f *FotffMocker) Steps(from, to string) (ret []string, err error) { + return f.steps, nil +} + +func (f *FotffMocker) GetNewer(cur string) (string, error) { + return "", nil +} + +func (f *FotffMocker) PkgDir(pkg string) string { + return pkg +} + +func (f *FotffMocker) TestCaseName() string { + return "MOCK_FAILED_TEST_CASE" +} + +func (f *FotffMocker) Last() string { + return f.steps[len(f.steps)-1] +} + +func TestFindOutTheFirstFail(t *testing.T) { + tests := []struct { + name string + mocker *FotffMocker + }{ + { + name: "0-1(X)", + mocker: NewFotffMocker(1, 1), + }, + { + name: "0-1(X)-2", + mocker: NewFotffMocker(2, 1), + }, + { + name: "0-1-2(X)", + mocker: NewFotffMocker(2, 2), + }, + { + name: "0-1(X)-2-3", + mocker: NewFotffMocker(3, 1), + }, + { + name: "0-1-2(X)-3", + mocker: NewFotffMocker(3, 2), + }, + { + name: "0-1-2-3(X)", + mocker: NewFotffMocker(3, 3), + }, + { + name: "0-1(X)-2-3-4", + mocker: NewFotffMocker(4, 1), + }, + { + name: "0-1-2(X)-3-4", + mocker: NewFotffMocker(4, 2), + }, + { + name: "0-1-2-3(X)-4", + mocker: NewFotffMocker(4, 3), + }, + { + name: "0-1-2-3-4(X)", + mocker: NewFotffMocker(4, 4), + }, + { + name: "0-1(X)-2-3-4-5", + mocker: NewFotffMocker(5, 1), + }, + { + name: "0-1-2(X)-3-4-5", + mocker: NewFotffMocker(5, 2), + }, + { + name: "0-1-2-3(X)-4-5", + mocker: NewFotffMocker(5, 3), + }, + { + name: "0-1-2-3-4(X)-5", + mocker: NewFotffMocker(5, 4), + }, + { + name: "0-1-2-3-4-5(X)", + mocker: NewFotffMocker(5, 5), + }, + { + name: "0-1-2...262143(X)...1048575", + mocker: NewFotffMocker(1048575, 262143), + }, + { + name: "0-1-2...262144(X)...1048575", + mocker: NewFotffMocker(1048575, 262144), + }, + { + name: "0-1-2...262145(X)...1048575", + mocker: NewFotffMocker(1048575, 262145), + }, + { + name: "0-1-2...262143(X)...1048576", + mocker: NewFotffMocker(1048576, 262143), + }, + { + name: "0-1-2...262144(X)...1048576", + mocker: NewFotffMocker(1048576, 262144), + }, + { + name: "0-1-2...262145(X)...1048576", + mocker: NewFotffMocker(1048576, 262145), + }, + { + name: "0-1-2...262143(X)...1048577", + mocker: NewFotffMocker(1048577, 262143), + }, + { + name: "0-1-2...262144(X)...1048577", + mocker: NewFotffMocker(1048577, 262144), + }, + { + name: "0-1-2...262145(X)...1048577", + mocker: NewFotffMocker(1048577, 262145), + }, + { + name: "0-1-2...1234567(X)...10000000", + mocker: NewFotffMocker(10000000, 1234567), + }, + { + name: "0-1-2...1234567(X)...100000001", + mocker: NewFotffMocker(10000001, 1234567), + }, + { + name: "0-1-2...7654321(X)...10000000", + mocker: NewFotffMocker(10000000, 7654321), + }, + { + name: "0-1-2...7654321(X)...10000001", + mocker: NewFotffMocker(10000001, 7654321), + }, + { + name: "0-1(X)-2...10000000", + mocker: NewFotffMocker(10000000, 1), + }, + { + name: "0-1(X)-2...10000001", + mocker: NewFotffMocker(10000001, 1), + }, + { + name: "0-1-2...10000000(X)", + mocker: NewFotffMocker(10000000, 10000000), + }, + { + name: "0-1-2...10000001(X)", + mocker: NewFotffMocker(10000001, 10000001), + }, + } + for i := 1; i <= 5; i++ { + res.Fake(i) + for _, tt := range tests { + t.Run(fmt.Sprintf("RES%d:%s", i, tt.name), func(t *testing.T) { + ret, err := FindOutTheFirstFail(tt.mocker, tt.mocker, tt.mocker.TestCaseName(), "0", tt.mocker.Last()) + if err != nil { + t.Errorf("err: expcect: , actual: %v", err) + } + expectIssue, _ := tt.mocker.LastIssue(tt.mocker.steps[tt.mocker.FirstFail-1]) + if ret != expectIssue { + t.Errorf("fotff result: expect: %s, actual: %s", expectIssue, ret) + } + }) + } + } +} diff --git a/tools/fotff/rec/record.go b/tools/fotff/rec/record.go new file mode 100644 index 0000000000000000000000000000000000000000..52894c68556b84dd25dd4aa78815c5eeaab26863 --- /dev/null +++ b/tools/fotff/rec/record.go @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rec + +import ( + "context" + "encoding/json" + "fotff/pkg" + "fotff/tester" + "fotff/utils" + "github.com/sirupsen/logrus" + "time" +) + +var Records = make(map[string]Record) + +func init() { + data, err := utils.ReadRuntimeData("records.json") + if err != nil { + return + } + if err := json.Unmarshal(data, &Records); err != nil { + logrus.Errorf("unmarshal records err: %v", err) + } +} + +func Save() { + data, err := json.MarshalIndent(Records, "", "\t") + if err != nil { + logrus.Errorf("marshal records err: %v", err) + return + } + if err := utils.WriteRuntimeData("records.json", data); err != nil { + logrus.Errorf("save records err: %v", err) + return + } + logrus.Infof("save records successfully") +} + +func HandleResults(t tester.Tester, dev string, pkgName string, results []tester.Result) []string { + var passes, fails []tester.Result + for _, result := range results { + switch result.Status { + case tester.ResultPass: + passes = append(passes, result) + case tester.ResultFail: + fails = append(fails, result) + } + } + handlePassResults(pkgName, passes) + return handleFailResults(t, dev, pkgName, fails) +} + +func handlePassResults(pkgName string, results []tester.Result) { + for _, result := range results { + logrus.Infof("recording [%s] as a success, the lastest success package is [%s]", result.TestCaseName, pkgName) + Records[result.TestCaseName] = Record{ + UpdateTime: time.Now().Format("2006-01-02 15:04:05"), + Status: tester.ResultPass, + LatestSuccessPkg: pkgName, + EarliestFailPkg: "", + FailIssueURL: "", + } + } +} + +func handleFailResults(t tester.Tester, dev string, pkgName string, results []tester.Result) []string { + var fotffTestCases []string + for _, result := range results { + if record, ok := Records[result.TestCaseName]; ok && record.Status != tester.ResultPass { + logrus.Warnf("test case %s had failed before, skip handle it", result.TestCaseName) + continue + } + status := tester.ResultFail + for i := 0; i < 3; i++ { + r, err := t.DoTestCase(dev, result.TestCaseName, context.TODO()) + if err != nil { + logrus.Errorf("failed to do test case %s: %v", result.TestCaseName, err) + continue + } + logrus.Infof("do testcase %s at %s done, result is %s", r.TestCaseName, dev, r.Status) + if r.Status == tester.ResultPass { + logrus.Warnf("testcase %s result is %s", r.TestCaseName, tester.ResultOccasionalFail) + status = tester.ResultOccasionalFail + break + } + } + if status == tester.ResultFail && Records[result.TestCaseName].LatestSuccessPkg != "" && Records[result.TestCaseName].EarliestFailPkg == "" { + fotffTestCases = append(fotffTestCases, result.TestCaseName) + } + Records[result.TestCaseName] = Record{ + UpdateTime: time.Now().Format("2006-01-02 15:04:05"), + Status: status, + LatestSuccessPkg: Records[result.TestCaseName].LatestSuccessPkg, + EarliestFailPkg: pkgName, + FailIssueURL: "", + } + } + return fotffTestCases +} + +func Analysis(m pkg.Manager, t tester.Tester, pkgName string, testcases []string) { + for i, testcase := range testcases { + record := Records[testcase] + logrus.Infof("%s failed, the lastest success package is [%s], earliest fail package is [%s], now finding out the first fail...", testcase, record.LatestSuccessPkg, pkgName) + issueURL, err := FindOutTheFirstFail(m, t, testcase, record.LatestSuccessPkg, pkgName, testcases[i+1:]...) + if err != nil { + logrus.Errorf("failed to find out the first fail issue, err: %v", err) + issueURL = err.Error() + } + logrus.Infof("recording %s as a failure, the lastest success package is [%s], the earliest fail package is [%s], fail issue URL is [%s]", testcase, record.LatestSuccessPkg, pkgName, issueURL) + Records[testcase] = Record{ + UpdateTime: time.Now().Format("2006-01-02 15:04:05"), + Status: tester.ResultFail, + LatestSuccessPkg: record.LatestSuccessPkg, + EarliestFailPkg: pkgName, + FailIssueURL: issueURL, + } + } +} diff --git a/tools/fotff/rec/report.go b/tools/fotff/rec/report.go new file mode 100644 index 0000000000000000000000000000000000000000..f8413f15441b179191c53522c26cce9a5ad05811 --- /dev/null +++ b/tools/fotff/rec/report.go @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rec + +import ( + "code.cloudfoundry.org/archiver/compressor" + "fmt" + "fotff/tester" + "fotff/utils" + "github.com/jedib0t/go-pretty/v6/table" + "github.com/jedib0t/go-pretty/v6/text" + "github.com/sirupsen/logrus" + "reflect" + "sort" +) + +const css = ` + + +` + +func Report(curPkg string, taskName string) { + subject := fmt.Sprintf("[%s] %s test report", curPkg, taskName) + rt := reflect.TypeOf(Record{}) + tb := table.NewWriter() + tb.SetIndexColumn(rt.NumField() + 1) + var row = table.Row{"test case"} + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + if f.IsExported() { + row = append(row, f.Tag.Get("col")) + } + } + tb.AppendHeader(row) + tb.SetRowPainter(func(row table.Row) text.Colors { + for _, col := range row { + if str, ok := col.(string); ok { + if str == tester.ResultFail { + return text.Colors{text.BgRed} + } else if str == tester.ResultOccasionalFail { + return text.Colors{text.BgYellow} + } + } + } + return nil + }) + var rows []table.Row + for k, rec := range Records { + var row = table.Row{k} + rv := reflect.ValueOf(rec) + for i := 0; i < rv.NumField(); i++ { + if rv.Field(i).CanInterface() { + row = append(row, rv.Field(i).Interface()) + } + } + rows = append(rows, row) + } + sort.Slice(rows, func(i, j int) bool { + return rows[i][0].(string) < rows[j][0].(string) + }) + tb.AppendRows(rows) + c := compressor.NewTgz() + var attrs []string + if utils.LogFile != nil { + if err := c.Compress(utils.LogFile.Name(), utils.LogFile.Name()+".tgz"); err != nil { + logrus.Errorf("failed to compress %s: %v", utils.LogFile.Name(), err) + } else { + attrs = append(attrs, utils.LogFile.Name()+".tgz") + } + } + if utils.StdoutFile != nil { + if err := c.Compress(utils.StdoutFile.Name(), utils.StdoutFile.Name()+".tgz"); err != nil { + logrus.Errorf("failed to compress %s: %v", utils.StdoutFile.Name(), err) + } else { + attrs = append(attrs, utils.StdoutFile.Name()+".tgz") + } + } + if err := utils.SendMail(subject, css+tb.RenderHTML(), attrs...); err != nil { + logrus.Errorf("failed to send report mail: %v", err) + return + } + logrus.Infof("send mail successfully") +} diff --git a/tools/fotff/rec/types.go b/tools/fotff/rec/types.go new file mode 100644 index 0000000000000000000000000000000000000000..4061172e4ff91f31965dc3bac4a9b17c7e253437 --- /dev/null +++ b/tools/fotff/rec/types.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rec + +type Record struct { + UpdateTime string `col:"update time"` + Status string `col:"status"` + LatestSuccessPkg string `col:"last success package"` + EarliestFailPkg string `col:"earliest fail package"` + FailIssueURL string `col:"fail issue url"` +} diff --git a/tools/fotff/res/res.go b/tools/fotff/res/res.go new file mode 100644 index 0000000000000000000000000000000000000000..74a4ef4366c6c1530a8a79ed6c7c981bdfdb3d2c --- /dev/null +++ b/tools/fotff/res/res.go @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package res + +import ( + "fmt" + "fotff/utils" + "strings" +) + +type Resources struct { + DeviceSnList string `key:"device_sn_list"` + AddrList string `key:"build_server_addr_list" default:"127.0.0.1:22"` + User string `key:"build_server_user" default:"root"` + Passwd string `key:"build_server_password" default:"root"` + // BuildWorkSpace must be absolute + BuildWorkSpace string `key:"build_server_workspace" default:"/root/fotff/build_workspace"` + devicePool chan string + serverPool chan string +} + +type BuildServerInfo struct { + Addr string + User string + Passwd string + WorkSpace string +} + +var res Resources + +func init() { + utils.ParseFromConfigFile("resources", &res) + snList := strings.Split(res.DeviceSnList, ",") + addrList := strings.Split(res.AddrList, ",") + res.devicePool = make(chan string, len(snList)) + for _, sn := range snList { + res.devicePool <- sn + } + res.serverPool = make(chan string, len(addrList)) + for _, addr := range addrList { + res.serverPool <- addr + } +} + +// Fake set 'n' fake packages and build servers. +// Just for test only. +func Fake(n int) { + var snList, addrList []string + for i := 0; i < n; i++ { + snList = append(snList, fmt.Sprintf("device%d", i)) + addrList = append(addrList, fmt.Sprintf("server%d", i)) + } + res.devicePool = make(chan string, len(snList)) + for _, sn := range snList { + res.devicePool <- sn + } + res.serverPool = make(chan string, len(addrList)) + for _, sn := range snList { + res.serverPool <- sn + } +} + +func Num() int { + if cap(res.devicePool) < cap(res.serverPool) { + return cap(res.devicePool) + } + return cap(res.serverPool) +} + +func DeviceList() []string { + return strings.Split(res.DeviceSnList, ",") +} + +func GetDevice() string { + return <-res.devicePool +} + +func ReleaseDevice(device string) { + res.devicePool <- device +} + +func GetBuildServer() BuildServerInfo { + addr := <-res.serverPool + return BuildServerInfo{ + Addr: addr, + User: res.User, + Passwd: res.Passwd, + WorkSpace: res.BuildWorkSpace, + } +} + +func ReleaseBuildServer(info BuildServerInfo) { + res.serverPool <- info.Addr +} diff --git a/tools/fotff/tester/manual/manual.go b/tools/fotff/tester/manual/manual.go new file mode 100644 index 0000000000000000000000000000000000000000..e039cc1a642b3daa8b3299252d971b0bf1913274 --- /dev/null +++ b/tools/fotff/tester/manual/manual.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package manual + +import ( + "context" + "fmt" + "fotff/tester" + "fotff/utils" + "github.com/sirupsen/logrus" + "math/rand" + "strings" + "sync" + "time" +) + +type Tester struct { + ResultLock sync.Mutex +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func NewTester() tester.Tester { + ret := &Tester{} + utils.ParseFromConfigFile("manual", ret) + return ret +} + +func (t *Tester) TaskName() string { + return "manual_test" +} + +func (t *Tester) Prepare(pkgDir string, device string, ctx context.Context) error { + return nil +} + +func (t *Tester) DoTestTask(deviceSN string, ctx context.Context) (ret []tester.Result, err error) { + return t.DoTestCases(deviceSN, []string{"default"}, ctx) +} + +func (t *Tester) DoTestCase(deviceSN, testCase string, ctx context.Context) (ret tester.Result, err error) { + if deviceSN == "" { + deviceSN = "default" + } + t.ResultLock.Lock() + defer t.ResultLock.Unlock() + var answer string + for { + fmt.Printf("please do testcase %s on device %s manually and type the test result, 'pass' or 'fail':\n", testCase, deviceSN) + if _, err := fmt.Scanln(&answer); err != nil { + logrus.Errorf("failed to scan result: %v", err) + continue + } + switch strings.ToUpper(strings.TrimSpace(answer)) { + case "PASS": + return tester.Result{TestCaseName: testCase, Status: tester.ResultPass}, nil + case "FAIL": + return tester.Result{TestCaseName: testCase, Status: tester.ResultFail}, nil + default: + } + } +} + +func (t *Tester) DoTestCases(deviceSN string, testcases []string, ctx context.Context) (ret []tester.Result, err error) { + for _, testcase := range testcases { + r, err := t.DoTestCase(deviceSN, testcase, ctx) + if err != nil { + return nil, err + } + ret = append(ret, r) + } + return ret, nil +} diff --git a/tools/fotff/tester/mock/mock.go b/tools/fotff/tester/mock/mock.go new file mode 100644 index 0000000000000000000000000000000000000000..93d9b89b94869a3116879c9bea675edb84ffe927 --- /dev/null +++ b/tools/fotff/tester/mock/mock.go @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mock + +import ( + "context" + "fotff/tester" + "github.com/sirupsen/logrus" +) + +type Tester struct{} + +func NewTester() tester.Tester { + return &Tester{} +} + +func (t *Tester) TaskName() string { + return "mock" +} + +func (t *Tester) Prepare(pkgDir string, device string, ctx context.Context) error { + return nil +} + +func (t *Tester) DoTestTask(device string, ctx context.Context) ([]tester.Result, error) { + logrus.Infof("TEST_001 pass") + logrus.Warnf("TEST_002 pass") + logrus.Warnf("TEST_003 pass") + return []tester.Result{ + {TestCaseName: "TEST_001", Status: tester.ResultPass}, + {TestCaseName: "TEST_002", Status: tester.ResultPass}, + {TestCaseName: "TEST_003", Status: tester.ResultPass}, + }, nil +} + +func (t *Tester) DoTestCase(device string, testCase string, ctx context.Context) (tester.Result, error) { + logrus.Warnf("%s pass", testCase) + return tester.Result{TestCaseName: testCase, Status: tester.ResultPass}, nil +} + +func (t *Tester) DoTestCases(device string, testcases []string, ctx context.Context) ([]tester.Result, error) { + var ret []tester.Result + for _, testcase := range testcases { + r, err := t.DoTestCase(device, testcase, ctx) + if err != nil { + return nil, err + } + ret = append(ret, r) + } + return ret, nil +} diff --git a/tools/fotff/tester/smoke/smoke.go b/tools/fotff/tester/smoke/smoke.go new file mode 100644 index 0000000000000000000000000000000000000000..3bc5cb11b9cd8a4fe6d70b8f9a9246705a0e81f8 --- /dev/null +++ b/tools/fotff/tester/smoke/smoke.go @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package smoke + +import ( + "context" + "crypto/md5" + "encoding/json" + "errors" + "fmt" + "fotff/tester" + "fotff/utils" + "github.com/sirupsen/logrus" + "math/rand" + "os" + "path/filepath" + "strconv" + "strings" + "time" +) + +type Tester struct { + Py string `key:"py"` + Config string `key:"config"` + AnswerPath string `key:"answer_path"` + SavePath string `key:"save_path"` + ToolsPath string `key:"tools_path"` +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func NewTester() tester.Tester { + ret := &Tester{} + utils.ParseFromConfigFile("smoke", ret) + return ret +} + +func (t *Tester) TaskName() string { + return "smoke_test" +} + +func (t *Tester) Prepare(pkgDir string, device string, ctx context.Context) error { + return nil +} + +func (t *Tester) DoTestTask(deviceSN string, ctx context.Context) (ret []tester.Result, err error) { + reportDir := fmt.Sprintf("%X", md5.Sum([]byte(fmt.Sprintf("%d", rand.Int())))) + if err := os.MkdirAll(filepath.Join(t.SavePath, reportDir), 0755); err != nil { + return nil, err + } + args := []string{t.Py, "--config", t.Config, "--answer_path", t.AnswerPath, "--save_path", filepath.Join(t.SavePath, reportDir), "--tools_path", t.ToolsPath} + if deviceSN != "" { + args = append(args, "--device_num", deviceSN) + } + if err := utils.ExecContext(ctx, "python", args...); err != nil { + if errors.Is(err, context.Canceled) { + return nil, err + } + logrus.Errorf("do test suite fail: %v", err) + return nil, err + } + return t.readReport(reportDir) +} + +func (t *Tester) DoTestCase(deviceSN, testCase string, ctx context.Context) (ret tester.Result, err error) { + reportDir := fmt.Sprintf("%X", md5.Sum([]byte(fmt.Sprintf("%d", rand.Int())))) + if err := os.MkdirAll(filepath.Join(t.SavePath, reportDir), 0755); err != nil { + return ret, err + } + args := []string{t.Py, "--config", t.Config, "--answer_path", t.AnswerPath, "--save_path", filepath.Join(t.SavePath, reportDir), "--tools_path", t.ToolsPath, "--test_num", testCase} + if deviceSN != "" { + args = append(args, "--device_num", deviceSN) + } + if err := utils.ExecContext(ctx, "python", args...); err != nil { + if errors.Is(err, context.Canceled) { + return ret, err + } + logrus.Errorf("do test case %s fail: %v", testCase, err) + return ret, err + } + r, err := t.readReport(reportDir) + if len(r) == 0 { + return ret, fmt.Errorf("read latest report err, no result found") + } + if r[0].TestCaseName != testCase { + return ret, fmt.Errorf("read latest report err, no matched result found") + } + logrus.Infof("do testcase %s at %s done, result is %s", r[0].TestCaseName, deviceSN, r[0].Status) + return r[0], nil +} + +func (t *Tester) DoTestCases(deviceSN string, testcases []string, ctx context.Context) (ret []tester.Result, err error) { + reportDir := fmt.Sprintf("%X", md5.Sum([]byte(fmt.Sprintf("%d", rand.Int())))) + if err := os.MkdirAll(filepath.Join(t.SavePath, reportDir), 0755); err != nil { + return nil, err + } + args := []string{t.Py, "--config", t.Config, "--answer_path", t.AnswerPath, "--save_path", filepath.Join(t.SavePath, reportDir), "--tools_path", t.ToolsPath, "--test_num", strings.Join(testcases, " ")} + if deviceSN != "" { + args = append(args, "--device_num", deviceSN) + } + if err := utils.ExecContext(ctx, "python", args...); err != nil { + if errors.Is(err, context.Canceled) { + return ret, err + } + logrus.Errorf("do test cases %v fail: %v", testcases, err) + return ret, err + } + return t.readReport(reportDir) +} + +func (t *Tester) readReport(reportDir string) (ret []tester.Result, err error) { + data, err := os.ReadFile(filepath.Join(t.SavePath, reportDir, "result.json")) + if err != nil { + logrus.Errorf("read report json fail: %v", err) + return nil, err + } + var result []struct { + TestCaseName int `json:"test_case_name"` + Status string `json:"status"` + } + err = json.Unmarshal(data, &result) + if err != nil { + logrus.Errorf("unmarshal report xml fail: %v", err) + return nil, err + } + for _, r := range result { + if r.Status == "pass" { + ret = append(ret, tester.Result{TestCaseName: strconv.Itoa(r.TestCaseName), Status: tester.ResultPass}) + } else { + ret = append(ret, tester.Result{TestCaseName: strconv.Itoa(r.TestCaseName), Status: tester.ResultFail}) + } + } + return ret, err +} diff --git a/tools/fotff/tester/tester.go b/tools/fotff/tester/tester.go new file mode 100644 index 0000000000000000000000000000000000000000..77eb783679d444ef58cbd26ec2c97d90663c9b9a --- /dev/null +++ b/tools/fotff/tester/tester.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tester + +import "context" + +type ResultStatus string + +const ( + ResultPass = `pass` + ResultOccasionalFail = `occasional_fail` + ResultFail = `fail` +) + +type Result struct { + TestCaseName string + Status ResultStatus +} + +type Tester interface { + // TaskName returns the name of task which DoTestTask execute. + TaskName() string + // Prepare do some test preparations for one certain package + Prepare(pkgDir string, device string, ctx context.Context) error + // DoTestTask do a full test on given device. + DoTestTask(device string, ctx context.Context) ([]Result, error) + // DoTestCase do a single testcase on given device. + DoTestCase(device string, testCase string, ctx context.Context) (Result, error) + // DoTestCases do testcases on given device. + DoTestCases(device string, testcases []string, ctx context.Context) ([]Result, error) +} + +type NewFunc func() Tester diff --git a/tools/fotff/tester/xdevice/xdevice.go b/tools/fotff/tester/xdevice/xdevice.go new file mode 100644 index 0000000000000000000000000000000000000000..4cc7d8d7c50c829c4a2aa01a295dbf0334ee9610 --- /dev/null +++ b/tools/fotff/tester/xdevice/xdevice.go @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdevice + +import ( + "context" + "crypto/md5" + "encoding/xml" + "errors" + "fmt" + "fotff/tester" + "fotff/utils" + "github.com/sirupsen/logrus" + "math/rand" + "os" + "path/filepath" + "strings" + "time" +) + +const enableTestModeScript = `mount -o rw,remount /; param set persist.ace.testmode.enabled 1; param set persist.sys.hilog.debug.on true; sed -i 's/enforcing/permissive/g' /system/etc/selinux/config; sync; reboot` + +type Tester struct { + Task string `key:"task" default:"acts"` + Config string `key:"config" default:"./config/user_config.xml"` + TestCasesPath string `key:"test_cases_path" default:"./testcases"` + ResourcePath string `key:"resource_path" default:"./resource"` +} + +type Report struct { + XMLName xml.Name `xml:"testsuites"` + TestSuite []struct { + TestCase []struct { + Name string `xml:"name,attr"` + Result string `xml:"result,attr"` + } `xml:"testcase"` + } `xml:"testsuite"` +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func NewTester() tester.Tester { + ret := &Tester{} + utils.ParseFromConfigFile("xdevice", ret) + return ret +} + +func (t *Tester) TaskName() string { + return t.Task +} + +func (t *Tester) Prepare(pkgDir string, device string, ctx context.Context) (err error) { + logrus.Info("for xdevice test, try to enable test mode...") + if err := utils.HdcShell(enableTestModeScript, device, ctx); err != nil { + return err + } + time.Sleep(20 * time.Second) // usually, it takes about 20s to reboot into OpenHarmony + if connected := utils.WaitHDC(device, ctx); !connected { + logrus.Errorf("enable test mode at device %s done, but boot unnormally, hdc connection fail", device) + return fmt.Errorf("enable test mode at device %s done, but boot unnormally, hdc connection fail", device) + } + time.Sleep(10 * time.Second) // wait 10s more to ensure system has been started completely + logrus.Infof("enable test mode at device %s successfully", device) + return nil +} + +func (t *Tester) DoTestTask(deviceSN string, ctx context.Context) (ret []tester.Result, err error) { + reportDir := fmt.Sprintf("%X", md5.Sum([]byte(fmt.Sprintf("%d", rand.Int())))) + args := []string{"-m", "xdevice", "run", t.Task, "-c", t.Config, "-tcpath", t.TestCasesPath, "-respath", t.ResourcePath, "-rp", reportDir} + if deviceSN != "" { + args = append(args, "-sn", deviceSN) + } + if err := utils.ExecContext(ctx, "python", args...); err != nil { + if errors.Is(err, context.Canceled) { + return nil, err + } + logrus.Errorf("do test suite fail: %v", err) + return nil, err + } + return t.readReport(reportDir) +} + +func (t *Tester) DoTestCase(deviceSN, testCase string, ctx context.Context) (ret tester.Result, err error) { + reportDir := fmt.Sprintf("%X", md5.Sum([]byte(fmt.Sprintf("%d", rand.Int())))) + args := []string{"-m", "xdevice", "run", "-l", testCase, "-c", t.Config, "-tcpath", t.TestCasesPath, "-respath", t.ResourcePath, "-rp", reportDir} + if deviceSN != "" { + args = append(args, "-sn", deviceSN) + } + if err := utils.ExecContext(ctx, "python", args...); err != nil { + if errors.Is(err, context.Canceled) { + return ret, err + } + logrus.Errorf("do test case %s fail: %v", testCase, err) + return ret, err + } + r, err := t.readReport(reportDir) + if len(r) == 0 { + return ret, fmt.Errorf("read latest report err, no result found") + } + if r[0].TestCaseName != testCase { + return ret, fmt.Errorf("read latest report err, no matched result found") + } + logrus.Infof("do testcase %s at %s done, result is %s", r[0].TestCaseName, deviceSN, r[0].Status) + return r[0], nil +} + +func (t *Tester) DoTestCases(deviceSN string, testcases []string, ctx context.Context) (ret []tester.Result, err error) { + reportDir := fmt.Sprintf("%X", md5.Sum([]byte(fmt.Sprintf("%d", rand.Int())))) + args := []string{"-m", "xdevice", "run", "-l", strings.Join(testcases, ";"), "-c", t.Config, "-tcpath", t.TestCasesPath, "-respath", t.ResourcePath, "-rp", reportDir} + if deviceSN != "" { + args = append(args, "-sn", deviceSN) + } + if err := utils.ExecContext(ctx, "python", args...); err != nil { + if errors.Is(err, context.Canceled) { + return ret, err + } + logrus.Errorf("do test cases %v fail: %v", testcases, err) + return ret, err + } + return t.readReport(reportDir) +} + +func (t *Tester) readReport(reportDir string) (ret []tester.Result, err error) { + data, err := os.ReadFile(filepath.Join("reports", reportDir, "summary_report.xml")) + if err != nil { + logrus.Errorf("read report xml fail: %v", err) + return nil, err + } + var report Report + err = xml.Unmarshal(data, &report) + if err != nil { + logrus.Errorf("unmarshal report xml fail: %v", err) + return nil, err + } + for _, s := range report.TestSuite { + for _, c := range s.TestCase { + var status tester.ResultStatus + if c.Result == "true" { + status = tester.ResultPass + } else { + status = tester.ResultFail + } + ret = append(ret, tester.Result{TestCaseName: c.Name, Status: status}) + } + } + return ret, err +} diff --git a/tools/fotff/utils/exec.go b/tools/fotff/utils/exec.go new file mode 100644 index 0000000000000000000000000000000000000000..76857e28435cecfe747874fdce630b452c10020a --- /dev/null +++ b/tools/fotff/utils/exec.go @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "context" + "errors" + "fmt" + "github.com/sirupsen/logrus" + "io" + "os" + "os/exec" + "time" +) + +func ExecContext(ctx context.Context, name string, args ...string) error { + ctx, fn := context.WithTimeout(ctx, 6*time.Hour) + defer fn() + if err := execContext(ctx, name, args...); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + logrus.Errorf("exec failed: %v, try again...", err) + return execContext(ctx, name, args...) + } + return nil +} + +func execContext(ctx context.Context, name string, args ...string) error { + cmdStr := append([]string{name}, args...) + logrus.Infof("cmd: %s", cmdStr) + cmd := exec.CommandContext(ctx, name, args...) + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + fmt.Printf("[%s] exec %s :\n", time.Now(), cmdStr) + go io.Copy(os.Stdout, stdout) + go io.Copy(os.Stderr, stderr) + return cmd.Wait() +} + +func ExecCombinedOutputContext(ctx context.Context, name string, args ...string) ([]byte, error) { + ctx, fn := context.WithTimeout(ctx, 6*time.Hour) + defer fn() + out, err := execCombinedOutputContext(ctx, name, args...) + if err != nil { + if errors.Is(err, context.Canceled) { + return out, err + } + logrus.Errorf("exec failed: %v, try again...", err) + return execCombinedOutputContext(ctx, name, args...) + } + return out, nil +} + +func execCombinedOutputContext(ctx context.Context, name string, args ...string) ([]byte, error) { + cmdStr := append([]string{name}, args...) + logrus.Infof("cmd: %s", cmdStr) + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + fmt.Printf("[%s] exec %s :\n", time.Now(), cmdStr) + return out, err +} + +func SleepContext(duration time.Duration, ctx context.Context) { + select { + case <-time.NewTimer(duration).C: + case <-ctx.Done(): + } +} diff --git a/tools/fotff/utils/hdc.go b/tools/fotff/utils/hdc.go new file mode 100644 index 0000000000000000000000000000000000000000..89b9e6173097c6e9973e1dee599fc94cdee20f16 --- /dev/null +++ b/tools/fotff/utils/hdc.go @@ -0,0 +1,81 @@ +package utils + +import ( + "context" + "errors" + "github.com/sirupsen/logrus" + "os/exec" + "strings" + "time" +) + +var hdc string + +func init() { + if hdc, _ = exec.LookPath("hdc"); hdc == "" { + hdc, _ = exec.LookPath("hdc_std") + } + if hdc == "" { + logrus.Panicf("can not find 'hdc', please install") + } +} + +func WaitHDC(device string, ctx context.Context) bool { + ctx, cancelFn := context.WithTimeout(ctx, 20*time.Second) + defer cancelFn() + for { + select { + case <-ctx.Done(): + return false + default: + } + ExecContext(ctx, hdc, "kill") + time.Sleep(time.Second) + ExecContext(ctx, hdc, "start") + time.Sleep(time.Second) + out, err := ExecCombinedOutputContext(ctx, hdc, "list", "targets") + if err != nil { + if errors.Is(err, context.Canceled) { + return false + } + logrus.Errorf("failed to list hdc targets: %s, %s", string(out), err) + continue + } + lines := strings.Fields(string(out)) + for _, dev := range lines { + if dev == "[Empty]" { + logrus.Warn("can not find any hdc targets") + break + } + if device == "" || dev == device { + return true + } + } + logrus.Infof("%s not found", device) + } +} + +func TryRebootToLoader(device string, ctx context.Context) error { + logrus.Infof("try to reboot %s to loader...", device) + defer time.Sleep(5 * time.Second) + if connected := WaitHDC(device, ctx); connected { + if device == "" { + return ExecContext(ctx, hdc, "shell", "reboot", "loader") + } else { + return ExecContext(ctx, hdc, "-t", device, "shell", "reboot", "loader") + } + } + if err := ctx.Err(); err != nil { + return err + } + logrus.Warn("can not find target hdc device, assume it has been in loader mode") + return nil +} + +func HdcShell(cmd, device string, ctx context.Context) error { + if device == "" { + return ExecContext(ctx, hdc, "shell", cmd) + } else { + return ExecContext(ctx, hdc, "-t", device, "shell", cmd) + } +} diff --git a/tools/fotff/utils/http.go b/tools/fotff/utils/http.go new file mode 100644 index 0000000000000000000000000000000000000000..e2ed194d54482542920e5f575b53ef05c40f09f4 --- /dev/null +++ b/tools/fotff/utils/http.go @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "bytes" + "fmt" + "github.com/sirupsen/logrus" + "io" + "net/http" + "time" +) + +func DoSimpleHttpReqRaw(method string, url string, body []byte, header map[string]string) (response *http.Response, err error) { + for i := 0; i < 3; i++ { + if response, err = doSimpleHttpReqImpl(method, url, body, header); err == nil { + return + } + time.Sleep(time.Second) + } + return +} + +func DoSimpleHttpReq(method string, url string, body []byte, header map[string]string) (ret []byte, err error) { + var resp *http.Response + for i := 0; i < 3; i++ { + if resp, err = doSimpleHttpReqImpl(method, url, body, header); err == nil { + ret, err = io.ReadAll(resp.Body) + resp.Body.Close() + return + } + time.Sleep(time.Second) + } + return +} + +func doSimpleHttpReqImpl(method string, url string, body []byte, header map[string]string) (response *http.Response, err error) { + logrus.Infof("%s %s", method, url) + req, err := http.NewRequest(method, url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + for k, v := range header { + req.Header.Set(k, v) + } + resp, err := proxyClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode >= 300 { + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode == http.StatusProxyAuthRequired || resp.StatusCode == http.StatusForbidden { + SwitchProxy() + } + logrus.Errorf("%s %s: code: %d body: %s", method, url, resp.StatusCode, string(data)) + return nil, fmt.Errorf("%s %s: code: %d body: %s", method, url, resp.StatusCode, string(data)) + } + return resp, nil +} diff --git a/tools/fotff/utils/ini.go b/tools/fotff/utils/ini.go new file mode 100644 index 0000000000000000000000000000000000000000..2680e7bb79a2098fe217bcff13dd1cc32ba2d724 --- /dev/null +++ b/tools/fotff/utils/ini.go @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "github.com/Unknwon/goconfig" + "github.com/sirupsen/logrus" + "reflect" +) + +// ParseFromConfigFile parse ini file and set values by the tag of fields. +// 'p' must be a pointer to the given structure, otherwise will panic. +// Only process its string fields and its sub structs. +func ParseFromConfigFile(section string, p any) { + conf, err := goconfig.LoadConfigFile("fotff.ini") + if err != nil { + logrus.Warnf("load config file err: %v", err) + } + rv := reflect.ValueOf(p) + rt := reflect.TypeOf(p) + for i := 0; i < rv.Elem().NumField(); i++ { + switch rt.Elem().Field(i).Type.Kind() { + case reflect.String: + key := rt.Elem().Field(i).Tag.Get("key") + if key == "" { + continue + } + var v string + if conf != nil { + v, err = conf.GetValue(section, key) + } + if conf == nil || err != nil { + v = rt.Elem().Field(i).Tag.Get("default") + } + rv.Elem().Field(i).SetString(v) + case reflect.Struct: + ParseFromConfigFile(section, rv.Elem().Field(i).Addr().Interface()) + } + } +} diff --git a/tools/fotff/utils/log.go b/tools/fotff/utils/log.go new file mode 100644 index 0000000000000000000000000000000000000000..9c9b8d64cb44a88d1bf62b0763e96c424edfc0bd --- /dev/null +++ b/tools/fotff/utils/log.go @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "fmt" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "runtime" + "strings" +) + +var LogFile *os.File +var StdoutFile *os.File +var osStdout, osStderr = os.Stdout, os.Stderr + +func init() { + if err := os.MkdirAll("logs", 0750); err != nil { + logrus.Errorf("can not make logs dir: %v", err) + return + } + logrus.SetOutput(os.Stdout) + logrus.SetReportCaller(true) + logrus.SetFormatter(&logrus.TextFormatter{ + ForceColors: true, + FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05", + CallerPrettyfier: func(f *runtime.Frame) (function string, file string) { + funcName := strings.Split(f.Function, ".") + fn := funcName[len(funcName)-1] + _, filename := filepath.Split(f.File) + return fmt.Sprintf("%s()", fn), fmt.Sprintf("%s:%d", filename, f.Line) + }, + }) +} + +func ResetLogOutput() { + logrus.Info("now log to os stdout...") + logrus.SetOutput(osStdout) + if LogFile != nil { + LogFile.Close() + } + if StdoutFile != nil { + StdoutFile.Close() + } + LogFile, StdoutFile, os.Stdout, os.Stderr = nil, nil, osStdout, osStderr +} + +func SetLogOutput(pkg string) { + file := filepath.Join("logs", pkg+".log") + var f *os.File + var err error + if _, err = os.Stat(file); err == nil { + f, err = os.OpenFile(file, os.O_RDWR|os.O_APPEND, 0666) + } else { + f, err = os.Create(file) + } + if err != nil { + logrus.Errorf("failed to open new log file %s: %v", file, err) + return + } + logrus.Infof("now log to %s", file) + logrus.SetOutput(f) + if LogFile != nil { + LogFile.Close() + } + LogFile = f + stdout := filepath.Join("logs", fmt.Sprintf("%s_stdout.log", pkg)) + if _, err = os.Stat(stdout); err == nil { + f, err = os.OpenFile(stdout, os.O_RDWR|os.O_APPEND, 0666) + } else { + f, err = os.Create(stdout) + } + if err != nil { + logrus.Errorf("failed to open new stdout log file %s: %v", stdout, err) + return + } + if StdoutFile != nil { + StdoutFile.Close() + } + StdoutFile, os.Stdout, os.Stderr = f, f, f + logrus.Infof("re-directing stdout and stderr to %s...", stdout) +} diff --git a/tools/fotff/utils/mail.go b/tools/fotff/utils/mail.go new file mode 100644 index 0000000000000000000000000000000000000000..5bf2485538501d76868342fa0e751974399033dd --- /dev/null +++ b/tools/fotff/utils/mail.go @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "crypto/tls" + "fmt" + "github.com/sirupsen/logrus" + "gopkg.in/gomail.v2" + "strconv" + "strings" +) + +type MailConfig struct { + Host string `key:"host" default:""` + Port string `key:"port" default:""` + port int + User string `key:"user" default:""` + Password string `key:"password" default:""` + From string `key:"from" default:""` + To string `key:"to" default:""` + toList []string +} + +var mailConfig MailConfig + +func init() { + ParseFromConfigFile("mail", &mailConfig) + if mailConfig.Host != "" { + var err error + if mailConfig.port, err = strconv.Atoi(mailConfig.Port); err != nil { + panic(fmt.Errorf("parse mail port err: %v", err)) + } + mailConfig.toList = strings.Split(mailConfig.To, ",") + } +} + +func SendMail(subject string, body string, attachments ...string) error { + if mailConfig.Host == "" { + logrus.Info("mail not configured, do nothing") + return nil + } + dail := gomail.NewDialer(mailConfig.Host, mailConfig.port, mailConfig.User, mailConfig.Password) + dail.TLSConfig = &tls.Config{InsecureSkipVerify: true, ServerName: mailConfig.Host} + msg := gomail.NewMessage() + msg.SetBody("text/html", body) + msg.SetHeader("From", mailConfig.From) + msg.SetHeader("To", mailConfig.toList...) + msg.SetHeader("Subject", subject) + for _, a := range attachments { + msg.Attach(a) + } + return dail.DialAndSend(msg) +} diff --git a/tools/fotff/utils/pprof.go b/tools/fotff/utils/pprof.go new file mode 100644 index 0000000000000000000000000000000000000000..8c505c4112dbba3d416bb1e274eafa2870f1c406 --- /dev/null +++ b/tools/fotff/utils/pprof.go @@ -0,0 +1,26 @@ +package utils + +import ( + "github.com/sirupsen/logrus" + "net" + "net/http" + _ "net/http/pprof" + "strconv" +) + +func EnablePprof() { + var cfg struct { + Enable string `key:"enable" default:"true"` + Port string `key:"port" default:"80"` + } + ParseFromConfigFile("pprof", &cfg) + if enable, _ := strconv.ParseBool(cfg.Enable); !enable { + return + } + server := &http.Server{Addr: net.JoinHostPort("localhost", cfg.Port)} + go func() { + if err := server.ListenAndServe(); err != nil { + logrus.Errorf("server.ListenAndServe returns error: %v", err) + } + }() +} diff --git a/tools/fotff/utils/proxy.go b/tools/fotff/utils/proxy.go new file mode 100644 index 0000000000000000000000000000000000000000..25f88c7607e714966d7af1a0aaef01b9b5ad6ab6 --- /dev/null +++ b/tools/fotff/utils/proxy.go @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "fmt" + "github.com/sirupsen/logrus" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +type ProxyConfig struct { + ServerList string `key:"server_list" default:""` + User string `key:"user" default:""` + Password string `key:"password" default:""` +} + +var proxyClient = http.DefaultClient +var ( + proxyUser string + proxyPassword string + proxyList []string + proxyIndex int + proxyLock sync.Mutex +) + +func init() { + var config ProxyConfig + ParseFromConfigFile("proxy", &config) + if len(config.ServerList) != 0 { + proxyList = strings.Split(config.ServerList, ",") + } + proxyUser = config.User + proxyPassword = config.Password + proxyIndex = len(proxyList) + SwitchProxy() + t := time.NewTicker(6 * time.Hour) + go func() { + <-t.C + proxyLock.Lock() + proxyIndex = len(proxyList) + proxyLock.Unlock() + }() +} + +func SwitchProxy() { + if len(proxyList) == 0 { + return + } + proxyLock.Lock() + defer proxyLock.Unlock() + proxyIndex++ + if proxyIndex >= len(proxyList) { + proxyIndex = 0 + } + var proxyURL *url.URL + var err error + logrus.Infof("switching proxy to %s", proxyList[proxyIndex]) + if proxyUser == "" { + proxyURL, err = url.Parse(fmt.Sprintf("http://%s", proxyList[proxyIndex])) + } else { + proxyURL, err = url.Parse(fmt.Sprintf("http://%s:%s@%s", proxyUser, url.QueryEscape(proxyPassword), proxyList[proxyIndex])) + } + if err != nil { + logrus.Errorf("failed to parse proxy url, err: %v", err) + } + proxyClient = &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + }, + } +} diff --git a/tools/fotff/utils/runtime.go b/tools/fotff/utils/runtime.go new file mode 100644 index 0000000000000000000000000000000000000000..40e499d399b43722e7f3283b40874697bf9f781f --- /dev/null +++ b/tools/fotff/utils/runtime.go @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "fmt" + "github.com/patrickmn/go-cache" + "os" + "path/filepath" + "time" +) + +var runtimeDir = `.fotff` + +var runtimeCache = cache.New(24*time.Hour, time.Hour) + +func sectionKey(section, key string) string { + return fmt.Sprintf("__%s__%s__", section, key) +} + +func init() { + if err := os.MkdirAll(runtimeDir, 0750); err != nil { + panic(err) + } + runtimeCache.LoadFile(filepath.Join(runtimeDir, "fotff.cache")) +} + +func CacheGet(section string, k string) (v any, found bool) { + return runtimeCache.Get(sectionKey(section, k)) +} + +func CacheSet(section string, k string, v any) error { + runtimeCache.Set(sectionKey(section, k), v, cache.DefaultExpiration) + return runtimeCache.SaveFile(filepath.Join(runtimeDir, "fotff.cache")) +} + +func WriteRuntimeData(name string, data []byte) error { + return os.WriteFile(filepath.Join(runtimeDir, name), data, 0640) +} + +func ReadRuntimeData(name string) ([]byte, error) { + return os.ReadFile(filepath.Join(runtimeDir, name)) +} diff --git a/tools/fotff/utils/ssh.go b/tools/fotff/utils/ssh.go new file mode 100644 index 0000000000000000000000000000000000000000..ad255d0ed5b45a25c9e2e521c959ce8bda4579b5 --- /dev/null +++ b/tools/fotff/utils/ssh.go @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "context" + "errors" + "fmt" + "github.com/pkg/sftp" + "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" + "io" + "os" + "path/filepath" + "time" +) + +func newSSHClient(addr string, user string, passwd string) (*ssh.Client, error) { + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{ssh.Password(passwd)}, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + config.SetDefaults() + return ssh.Dial("tcp", addr, config) +} + +func RunCmdViaSSHContext(ctx context.Context, addr string, user string, passwd string, cmd string) (err error) { + ctx, fn := context.WithTimeout(ctx, 6*time.Hour) + defer fn() + if err := RunCmdViaSSHContextNoRetry(ctx, addr, user, passwd, cmd); err != nil { + if errors.Is(err, context.Canceled) { + return err + } + logrus.Errorf("exec cmd via SSH at %s failed: %v, try again...", addr, err) + return RunCmdViaSSHContextNoRetry(ctx, addr, user, passwd, cmd) + } + return nil +} + +func RunCmdViaSSHContextNoRetry(ctx context.Context, addr string, user string, passwd string, cmd string) (err error) { + exit := make(chan struct{}) + client, err := newSSHClient(addr, user, passwd) + if err != nil { + logrus.Errorf("new SSH client to %s err: %v", addr, err) + return err + } + defer client.Close() + session, err := client.NewSession() + if err != nil { + return err + } + defer func() { + select { + case <-ctx.Done(): + err = ctx.Err() + default: + } + }() + defer close(exit) + go func() { + select { + case <-ctx.Done(): + case <-exit: + } + session.Close() + }() + logrus.Infof("run at %s: %s", addr, cmd) + stdin, err := session.StdinPipe() + if err != nil { + return err + } + defer stdin.Close() + stdout, err := session.StdoutPipe() + if err != nil { + return err + } + stderr, err := session.StderrPipe() + if err != nil { + return err + } + if err := session.Shell(); err != nil { + return err + } + cmd = fmt.Sprintf("%s\nexit $?\n", cmd) + go stdin.Write([]byte(cmd)) + go io.Copy(os.Stdout, stdout) + go io.Copy(os.Stderr, stderr) + fmt.Printf("[%s] exec at %s %s :\n", time.Now(), addr, cmd) + return session.Wait() +} + +type Direct string + +const ( + Download Direct = "download" + Upload Direct = "upload" +) + +func TransFileViaSSH(verb Direct, addr string, user string, passwd string, remoteFile string, localFile string) error { + c, err := newSSHClient(addr, user, passwd) + if err != nil { + logrus.Errorf("new SSH client to %s err: %v", addr, err) + return err + } + defer c.Close() + client, err := sftp.NewClient(c) + if err != nil { + logrus.Errorf("new SFTP client to %s err: %v", addr, err) + return err + } + defer client.Close() + var prep string + var src, dst io.ReadWriteCloser + if verb == Download { + prep = "to" + if src, err = client.Open(remoteFile); err != nil { + return fmt.Errorf("open remote file %s at %s err: %v", remoteFile, addr, err) + } + defer src.Close() + os.RemoveAll(localFile) + os.MkdirAll(filepath.Dir(localFile), 0755) + if dst, err = os.Create(localFile); err != nil { + return fmt.Errorf("create local file err: %v", err) + } + defer dst.Close() + } else { + prep = "from" + if src, err = os.Open(localFile); err != nil { + return fmt.Errorf("open local file err: %v", err) + } + defer src.Close() + client.Remove(remoteFile) + client.MkdirAll(filepath.Dir(remoteFile)) + if dst, err = client.Create(remoteFile); err != nil { + return fmt.Errorf("create remote file %s at %s err: %v", remoteFile, addr, err) + } + defer dst.Close() + } + logrus.Infof("%sing %s at %s %s %s...", verb, remoteFile, addr, prep, localFile) + t1 := time.Now() + n, err := io.CopyBuffer(dst, src, make([]byte, 32*1024*1024)) + if err != nil { + logrus.Errorf("%s %s at %s %s %s err: %v", verb, remoteFile, addr, prep, localFile, err) + return err + } + t2 := time.Now() + cost := t2.Sub(t1).Seconds() + logrus.Infof("%s %s at %s %s %s done, size: %d cost: %.2fs speed: %.2fMB/s", verb, remoteFile, addr, prep, localFile, n, cost, float64(n)/cost/1024/1024) + return nil +} diff --git a/tools/fotff/vcs/gitee/branch.go b/tools/fotff/vcs/gitee/branch.go new file mode 100644 index 0000000000000000000000000000000000000000..10c196ffcca4f1f6fd819cdd51ad58b9f2c78be5 --- /dev/null +++ b/tools/fotff/vcs/gitee/branch.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee + +import ( + "encoding/json" + "fmt" + "fotff/utils" + "net/http" +) + +type BranchResp struct { + Name string `json:"name"` + Commit *Commit `json:"commit"` +} + +func GetBranch(owner, repo, branch string) (*BranchResp, error) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/branches/%s", owner, repo, branch) + resp, err := utils.DoSimpleHttpReq(http.MethodGet, url, nil, nil) + if err != nil { + return nil, err + } + var branchResp BranchResp + if err := json.Unmarshal(resp, &branchResp); err != nil { + return nil, err + } + return &branchResp, nil +} diff --git a/tools/fotff/vcs/gitee/commit.go b/tools/fotff/vcs/gitee/commit.go new file mode 100644 index 0000000000000000000000000000000000000000..21c3c96bff641a980b3599a8dc5e535740e70f26 --- /dev/null +++ b/tools/fotff/vcs/gitee/commit.go @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee + +import ( + "encoding/json" + "fmt" + "fotff/utils" + "net/http" +) + +func GetCommit(owner, repo, id string) (*Commit, error) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/commits/%s", owner, repo, id) + var resp []byte + if c, found := utils.CacheGet("gitee", url); found { + resp = c.([]byte) + } else { + var err error + resp, err = utils.DoSimpleHttpReq(http.MethodGet, url, nil, nil) + if err != nil { + return nil, err + } + utils.CacheSet("gitee", url, resp) + } + var commitResp Commit + if err := json.Unmarshal(resp, &commitResp); err != nil { + return nil, err + } + commitResp.Owner = owner + commitResp.Repo = repo + return &commitResp, nil +} diff --git a/tools/fotff/vcs/gitee/compare.go b/tools/fotff/vcs/gitee/compare.go new file mode 100644 index 0000000000000000000000000000000000000000..4271d2423804b36edfd939522cf34bc387d6f518 --- /dev/null +++ b/tools/fotff/vcs/gitee/compare.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee + +import ( + "encoding/json" + "fmt" + "fotff/utils" + "net/http" + "time" +) + +type CompareParam struct { + Head string + Base string + Repo string + Owner string +} + +type CompareResp struct { + Commits []*Commit `json:"commits"` +} + +type Commit struct { + CommitExtend `json:"-"` + URL string `json:"url"` + SHA string `json:"sha"` + Commit struct { + Committer struct { + Date string `json:"date"` + } `json:"committer"` + Message string `json:"message"` + } `json:"commit"` + Parents []struct { + SHA string `json:"sha"` + URL string `json:"url"` + } `json:"parents"` + Files []struct { + Filename string `json:"filename"` + Status string `json:"status"` + Patch string `json:"patch,omitempty"` + } `json:"files,omitempty"` +} + +type CommitExtend struct { + Owner string + Repo string +} + +func GetLatestMRBefore(owner, repo, branch string, before string) (ret *Commit, err error) { + branchResp, err := GetBranch(owner, repo, branch) + if err != nil { + return nil, err + } + head := branchResp.Commit + head.Owner = owner + head.Repo = repo + for head.Commit.Committer.Date > before { + if head, err = GetCommit(owner, repo, head.Parents[0].SHA); err != nil { + return nil, err + } + } + return head, nil +} + +func GetBetweenTimeMRs(owner, repo, branch string, from, to time.Time) (ret []*Commit, err error) { + branchResp, err := GetBranch(owner, repo, branch) + if err != nil { + return nil, err + } + fromStr := from.UTC().Format(time.RFC3339) + toStr := to.UTC().Format(time.RFC3339) + head := branchResp.Commit + head.Owner = owner + head.Repo = repo + for head.Commit.Committer.Date > fromStr { + if head.Commit.Committer.Date < toStr { + ret = append(ret, head) + } + if head, err = GetCommit(owner, repo, head.Parents[0].SHA); err != nil { + return nil, err + } + } + return ret, nil +} + +func GetBetweenMRs(param CompareParam) ([]*Commit, error) { + commits, err := GetBetweenCommits(param) + if err != nil { + return nil, err + } + var ret []*Commit + head := param.Head + for head != param.Base { + for _, commit := range commits { + if commit.SHA != head { + continue + } + commit.Owner = param.Owner + commit.Repo = param.Repo + ret = append(ret, commit) + head = commit.Parents[0].SHA + } + } + return ret, nil +} + +func GetBetweenCommits(param CompareParam) ([]*Commit, error) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/compare/%s...%s", param.Owner, param.Repo, param.Base, param.Head) + var resp []byte + if c, found := utils.CacheGet("gitee", url); found { + resp = c.([]byte) + } else { + var err error + resp, err = utils.DoSimpleHttpReq(http.MethodGet, url, nil, nil) + if err != nil { + return nil, err + } + utils.CacheSet("gitee", url, resp) + } + var compareResp CompareResp + if err := json.Unmarshal(resp, &compareResp); err != nil { + return nil, err + } + return compareResp.Commits, nil +} diff --git a/tools/fotff/vcs/gitee/issue.go b/tools/fotff/vcs/gitee/issue.go new file mode 100644 index 0000000000000000000000000000000000000000..48a2c04834e31567ede2caef51bcddb6ab725226 --- /dev/null +++ b/tools/fotff/vcs/gitee/issue.go @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee + +import ( + "encoding/json" + "fmt" + "fotff/utils" + "net/http" +) + +type PRIssueResp struct { + URL string `json:"html_url"` +} + +func GetMRIssueURL(owner string, repo string, num int) ([]string, error) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/pulls/%d/issues", owner, repo, num) + var resp []byte + if c, found := utils.CacheGet("gitee", url); found { + resp = c.([]byte) + } else { + var err error + resp, err = utils.DoSimpleHttpReq(http.MethodGet, url, nil, nil) + if err != nil { + return nil, err + } + utils.CacheSet("gitee", url, resp) + } + var prIssues []PRIssueResp + if err := json.Unmarshal(resp, &prIssues); err != nil { + return nil, err + } + ret := make([]string, len(prIssues)) + for i, issue := range prIssues { + ret[i] = issue.URL + } + return ret, nil +} diff --git a/tools/fotff/vcs/manifest.go b/tools/fotff/vcs/manifest.go new file mode 100644 index 0000000000000000000000000000000000000000..93c3caac25f13dbbfa75150f1e400b315f4771d6 --- /dev/null +++ b/tools/fotff/vcs/manifest.go @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vcs + +import ( + "crypto/md5" + "encoding/xml" + "fmt" + "github.com/sirupsen/logrus" + "os" + "sort" +) + +type Manifest struct { + XMLName xml.Name `xml:"manifest"` + Remote Remote `xml:"remote"` + Default Default `xml:"default"` + Projects []Project `xml:"project"` +} + +type Remote struct { + Name string `xml:"name,attr"` + Fetch string `xml:"fetch,attr"` + Review string `xml:"review,attr"` +} + +type Default struct { + Remote string `xml:"remote,attr"` + Revision string `xml:"revision,attr"` + SyncJ string `xml:"sync-j,attr"` +} + +type Project struct { + XMLName xml.Name `xml:"project"` + Name string `xml:"name,attr"` + Path string `xml:"path,attr,omitempty"` + Revision string `xml:"revision,attr"` + Remote string `xml:"remote,attr,omitempty"` + CloneDepth string `xml:"clone-depth,attr,omitempty"` + LinkFile []LinkFile `xml:"linkfile,omitempty"` +} + +type LinkFile struct { + Src string `xml:"src,attr"` + Dest string `xml:"dest,attr"` +} + +type ProjectUpdate struct { + P1, P2 *Project +} + +func (p *Project) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("<%s>", p.Name) +} + +func (p *Project) StructureDiff(p2 *Project) bool { + if p == nil && p2 != nil || p != nil && p2 == nil { + return true + } + if p == nil && p2 == nil { + return false + } + return p.Name != p2.Name || p.Path != p2.Path || p.Remote != p2.Remote +} + +func (p *Project) Equals(p2 *Project) bool { + return p.Name == p2.Name && p.Path == p2.Path && p.Remote == p2.Remote && p.Revision == p2.Revision +} + +func ParseManifestFile(file string) (*Manifest, error) { + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + var m Manifest + err = xml.Unmarshal(data, &m) + return &m, err +} + +func (m *Manifest) WriteFile(filePath string) error { + data, err := xml.MarshalIndent(m, "", " ") + if err != nil { + return err + } + data = append([]byte(xml.Header), data...) + return os.WriteFile(filePath, data, 0640) +} + +func GetRepoUpdates(m1, m2 *Manifest) (updates []ProjectUpdate, err error) { + if _, err := m1.Standardize(); err != nil { + return nil, err + } + if _, err := m2.Standardize(); err != nil { + return nil, err + } + var j int + for i := 0; i < len(m1.Projects); { + if m2.Projects[j].Name == m1.Projects[i].Name { + if !m1.Projects[i].Equals(&m2.Projects[j]) { + logrus.Infof("%v changes", &m1.Projects[i]) + updates = append(updates, ProjectUpdate{ + P1: &m1.Projects[i], + P2: &m2.Projects[j], + }) + } + i++ + j++ + } else if m2.Projects[j].Name > m1.Projects[i].Name { + logrus.Infof("%v removed", &m1.Projects[i]) + updates = append(updates, ProjectUpdate{ + P1: &m1.Projects[i], + P2: nil, + }) + i++ + } else { // m2.Projects[j].Name < m1.Projects[i].Name + logrus.Infof("%v added", &m2.Projects[j]) + updates = append(updates, ProjectUpdate{ + P1: nil, + P2: &m2.Projects[j], + }) + j++ + } + } + return +} + +func (m *Manifest) UpdateManifestProject(name, path, remote, revision string, add bool) { + if name == "manifest" { + return + } + for i, p := range m.Projects { + if p.Name == name { + if path != "" { + m.Projects[i].Path = path + } + if remote != "" { + m.Projects[i].Remote = remote + } + if revision != "" { + m.Projects[i].Revision = revision + } + return + } + } + if add { + m.Projects = append(m.Projects, Project{Name: name, Path: path, Revision: revision, Remote: remote}) + } +} + +func (m *Manifest) RemoveManifestProject(name string) { + for i, p := range m.Projects { + if p.Name == name { + m.Projects = append(m.Projects[:i], m.Projects[i:]...) + return + } + } +} + +func (m *Manifest) Standardize() (string, error) { + sort.Slice(m.Projects, func(i, j int) bool { + return m.Projects[i].Name < m.Projects[j].Name + }) + data, err := xml.MarshalIndent(m, "", " ") + if err != nil { + return "", err + } + data = append([]byte(xml.Header), data...) + sumByte := md5.Sum(data) + return fmt.Sprintf("%X", sumByte), nil +}