From 4354fa87fdac3d4d9ef4d4a39e46411d477a89c1 Mon Sep 17 00:00:00 2001 From: huangshan Date: Mon, 10 Apr 2023 15:34:40 +0800 Subject: [PATCH] refactor pkg to support more devices Signed-off-by: huangshan Change-Id: Ibe479d5840a10130a98ad2f3cd82bf888a95215e --- tools/fotff/main.go | 18 +- tools/fotff/pkg/dayu200/dayu200.go | 172 ++-- tools/fotff/pkg/gitee_build/gitee_build.go | 74 ++ .../pkg/{dayu200 => gitee_common}/build.go | 75 +- tools/fotff/pkg/gitee_common/common.go | 151 ++++ .../{dayu200 => gitee_common}/get_newer_ci.go | 73 +- tools/fotff/pkg/gitee_common/get_newer_dir.go | 45 ++ .../pkg/gitee_common/get_newer_or_fail.go | 111 +++ .../pkg/{dayu200 => gitee_common}/steps_ci.go | 12 +- .../{dayu200 => gitee_common}/steps_gitee.go | 732 +++++++++--------- .../steps_gitee_test.go | 2 +- .../manifest_tag.xml | 0 .../manifest_tag.xml | 0 .../manifest_tag.xml | 0 .../manifest_tag.xml | 0 .../manifest_tag.xml | 0 .../manifest_tag.xml | 0 tools/fotff/pkg/pkg.go | 26 - tools/fotff/rec/fotff.go | 13 +- .../tester/pkg_available/pkg_available.go | 81 ++ tools/fotff/utils/http.go | 12 +- tools/fotff/utils/ini.go | 17 + 22 files changed, 1007 insertions(+), 607 deletions(-) create mode 100644 tools/fotff/pkg/gitee_build/gitee_build.go rename tools/fotff/pkg/{dayu200 => gitee_common}/build.go (55%) create mode 100644 tools/fotff/pkg/gitee_common/common.go rename tools/fotff/pkg/{dayu200 => gitee_common}/get_newer_ci.go (63%) create mode 100644 tools/fotff/pkg/gitee_common/get_newer_dir.go create mode 100644 tools/fotff/pkg/gitee_common/get_newer_or_fail.go rename tools/fotff/pkg/{dayu200 => gitee_common}/steps_ci.go (95%) rename tools/fotff/pkg/{dayu200 => gitee_common}/steps_gitee.go (94%) rename tools/fotff/pkg/{dayu200 => gitee_common}/steps_gitee_test.go (99%) rename tools/fotff/pkg/{dayu200 => gitee_common}/testdata/version-Daily_Version-dayu200-20221201_080109-dayu200/manifest_tag.xml (100%) rename tools/fotff/pkg/{dayu200 => gitee_common}/testdata/version-Daily_Version-dayu200-20221201_100141-dayu200/manifest_tag.xml (100%) rename tools/fotff/pkg/{dayu200 => gitee_common}/testdata/version-Daily_Version-dayu200-20221213_110027-dayu200/manifest_tag.xml (100%) rename tools/fotff/pkg/{dayu200 => gitee_common}/testdata/version-Daily_Version-dayu200-20221213_140150-dayu200/manifest_tag.xml (100%) rename tools/fotff/pkg/{dayu200 => gitee_common}/testdata/version-Daily_Version-dayu200-20221214_100124-dayu200/manifest_tag.xml (100%) rename tools/fotff/pkg/{dayu200 => gitee_common}/testdata/version-Daily_Version-dayu200-20221214_110125-dayu200/manifest_tag.xml (100%) create mode 100644 tools/fotff/tester/pkg_available/pkg_available.go diff --git a/tools/fotff/main.go b/tools/fotff/main.go index a5f4fde..4bb156f 100644 --- a/tools/fotff/main.go +++ b/tools/fotff/main.go @@ -19,6 +19,7 @@ import ( "context" "fotff/pkg" "fotff/pkg/dayu200" + "fotff/pkg/gitee_build" "fotff/pkg/mock" "fotff/rec" "fotff/res" @@ -26,6 +27,7 @@ import ( "fotff/tester/common" "fotff/tester/manual" testermock "fotff/tester/mock" + "fotff/tester/pkg_available" "fotff/tester/smoke" "fotff/tester/xdevice" "fotff/utils" @@ -37,16 +39,18 @@ import ( ) var newPkgMgrFuncs = map[string]pkg.NewFunc{ - "mock": mock.NewManager, - "dayu200": dayu200.NewManager, + "mock": mock.NewManager, + "dayu200": dayu200.NewManager, + "gitee_build": gitee_build.NewManager, } var newTesterFuncs = map[string]tester.NewFunc{ - "mock": testermock.NewTester, - "manual": manual.NewTester, - "common": common.NewTester, - "xdevice": xdevice.NewTester, - "smoke": smoke.NewTester, + "mock": testermock.NewTester, + "manual": manual.NewTester, + "common": common.NewTester, + "xdevice": xdevice.NewTester, + "smoke": smoke.NewTester, + "pkg_available": pkg_available.NewTester, } var rootCmd *cobra.Command diff --git a/tools/fotff/pkg/dayu200/dayu200.go b/tools/fotff/pkg/dayu200/dayu200.go index 17ae273..b02a9de 100644 --- a/tools/fotff/pkg/dayu200/dayu200.go +++ b/tools/fotff/pkg/dayu200/dayu200.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Huawei Device Co., Ltd. + * Copyright (c) 2022-2023 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -16,38 +16,62 @@ package dayu200 import ( - "code.cloudfoundry.org/archiver/extractor" "context" - "fmt" "fotff/pkg" + "fotff/pkg/gitee_common" "fotff/res" "fotff/utils" "github.com/sirupsen/logrus" - "os" - "path/filepath" "strconv" "strings" - "time" ) type Manager struct { - ArchiveDir string `key:"archive_dir" default:"."` - FromCI string `key:"download_from_ci" default:"false"` - Workspace string `key:"workspace" default:"."` + ArchiveDir string `key:"archive_dir" default:"archive"` + WatchCI string `key:"watch_ci" default:"false"` + Workspace string `key:"workspace" default:"workspace"` Branch string `key:"branch" default:"master"` FlashTool string `key:"flash_tool" default:"python"` LocationIDList string `key:"location_id_list"` - locations map[string]string - fromCI bool + + *gitee_common.Manager + locations map[string]string +} + +// These commands are copied from ci project. +const ( + preCompileCMD = `rm -rf prebuilts/clang/ohos/darwin-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/windows-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/linux-x86_64/clang-480513;bash build/prebuilts_download.sh` + // compileCMD is copied from ci project and trim useless build-target 'make_test' to enhance build efficiency. + compileCMD = `echo 'start' && export NO_DEVTOOL=1 && export CCACHE_LOG_SUFFIX="dayu200-arm32" && export CCACHE_NOHASHDIR="true" && export CCACHE_SLOPPINESS="include_file_ctime" && ./build.sh --product-name rk3568 --ccache --build-target make_all --gn-args enable_notice_collection=false` +) + +// This list is copied from ci project. Some of them are not available, has been annotated. +var imgList = []string{ + "out/rk3568/packages/phone/images/MiniLoaderAll.bin", + "out/rk3568/packages/phone/images/boot_linux.img", + "out/rk3568/packages/phone/images/parameter.txt", + "out/rk3568/packages/phone/images/system.img", + "out/rk3568/packages/phone/images/uboot.img", + "out/rk3568/packages/phone/images/userdata.img", + "out/rk3568/packages/phone/images/vendor.img", + "out/rk3568/packages/phone/images/resource.img", + "out/rk3568/packages/phone/images/config.cfg", + "out/rk3568/packages/phone/images/ramdisk.img", + // "out/rk3568/packages/phone/images/chipset.img", + "out/rk3568/packages/phone/images/sys_prod.img", + "out/rk3568/packages/phone/images/chip_prod.img", + "out/rk3568/packages/phone/images/updater.img", + // "out/rk3568/packages/phone/updater/bin/updater_binary", } func NewManager() pkg.Manager { var ret Manager utils.ParseFromConfigFile("dayu200", &ret) - var err error - if ret.fromCI, err = strconv.ParseBool(ret.FromCI); err != nil { - logrus.Panicf("can not parse 'download_from_ci', please check") + watchCI, err := strconv.ParseBool(ret.WatchCI) + if err != nil { + logrus.Panicf("can not parse 'watch_ci', please check") } + ret.Manager = gitee_common.NewManager("dayu200", ret.Branch, ret.ArchiveDir, ret.Workspace, watchCI) devs := res.DeviceList() locs := strings.Split(ret.LocationIDList, ",") if len(devs) != len(locs) { @@ -57,125 +81,23 @@ func NewManager() pkg.Manager { for i, loc := range locs { ret.locations[devs[i]] = loc } - go ret.cleanupOutdated() return &ret } -func (m *Manager) cleanupOutdated() { - t := time.NewTicker(24 * time.Hour) - for { - <-t.C - es, err := os.ReadDir(m.Workspace) - if err != nil { - logrus.Errorf("can not read %s: %v", m.Workspace, err) - continue - } - for _, e := range es { - if !e.IsDir() { - continue - } - path := filepath.Join(m.Workspace, e.Name()) - info, err := e.Info() - if err != nil { - logrus.Errorf("can not read %s info: %v", path, err) - continue - } - if time.Now().Sub(info.ModTime()) > 7*24*time.Hour { - logrus.Warnf("%s outdated, cleanning up its contents...", path) - m.cleanupPkgFiles(path) - } - } - } -} - -func (m *Manager) cleanupPkgFiles(path string) { - es, err := os.ReadDir(path) - if err != nil { - logrus.Errorf("can not read %s: %v", path, err) - return - } - for _, e := range es { - if e.Name() == "manifest_tag.xml" || e.Name() == "__last_issue__" { - continue - } - if err := os.RemoveAll(filepath.Join(path, e.Name())); err != nil { - logrus.Errorf("remove %s fail: %v", filepath.Join(path, e.Name()), err) - } - } -} - // Flash function implements pkg.Manager. Flash images in the 'pkg' directory to the given device. // If not all necessary images are available in the 'pkg' directory, will build them. func (m *Manager) Flash(device string, pkg string, ctx context.Context) error { logrus.Infof("now flash %s", pkg) - if !m.pkgAvailable(pkg) { - logrus.Infof("%s is not available", pkg) - if err := m.build(pkg, false, ctx); err != nil { - logrus.Errorf("build pkg %s err: %v", pkg, err) - logrus.Infof("build pkg %s again...", pkg) - if err = m.build(pkg, true, ctx); err != nil { - logrus.Errorf("build pkg %s err: %v", pkg, err) - return err - } - } + buildConfig := gitee_common.BuildConfig{ + Pkg: pkg, + PreCompileCMD: preCompileCMD, + CompileCMD: compileCMD, + ImgList: imgList, + } + if err := m.Build(buildConfig, ctx); err != nil { + logrus.Errorf("build %s fail, err: %v", pkg, err) + return err } logrus.Infof("%s is available now, start to flash it", pkg) return m.flashDevice(device, pkg, ctx) } - -func (m *Manager) Steps(from, to string) (pkgs []string, err error) { - if from == to { - return nil, fmt.Errorf("steps err: 'from' %s and 'to' %s are the same", from, to) - } - if c, found := utils.CacheGet("dayu200_steps", from+"__to__"+to); found { - logrus.Infof("steps from %s to %s are cached", from, to) - logrus.Infof("steps: %v", c.([]string)) - return c.([]string), nil - } - if pkgs, err = m.stepsFromGitee(from, to); err != nil { - logrus.Errorf("failed to gen steps from gitee, err: %v", err) - logrus.Warnf("fallback to getting steps from CI...") - if pkgs, err = m.stepsFromCI(from, to); err != nil { - return pkgs, err - } - return pkgs, nil - } - utils.CacheSet("dayu200_steps", from+"__to__"+to, pkgs) - return pkgs, nil -} - -func (m *Manager) LastIssue(pkg string) (string, error) { - data, err := os.ReadFile(filepath.Join(m.Workspace, pkg, "__last_issue__")) - return string(data), err -} - -func (m *Manager) GetNewer(cur string) (string, error) { - var newFile string - if m.fromCI { - newFile = m.getNewerFromCI(cur + ".tar.gz") - } else { - newFile = pkg.GetNewerFileFromDir(m.ArchiveDir, cur+".tar.gz", func(files []os.DirEntry, i, j int) bool { - ti, _ := getPackageTime(files[i].Name()) - tj, _ := getPackageTime(files[j].Name()) - return ti.Before(tj) - }) - } - ex := extractor.NewTgz() - dirName := newFile - for filepath.Ext(dirName) != "" { - dirName = strings.TrimSuffix(dirName, filepath.Ext(dirName)) - } - dir := filepath.Join(m.Workspace, dirName) - if _, err := os.Stat(dir); err == nil { - return dirName, nil - } - logrus.Infof("extracting %s to %s...", filepath.Join(m.ArchiveDir, newFile), dir) - if err := ex.Extract(filepath.Join(m.ArchiveDir, newFile), dir); err != nil { - return dirName, err - } - return dirName, nil -} - -func (m *Manager) PkgDir(pkg string) string { - return filepath.Join(m.Workspace, pkg) -} diff --git a/tools/fotff/pkg/gitee_build/gitee_build.go b/tools/fotff/pkg/gitee_build/gitee_build.go new file mode 100644 index 0000000..ad24018 --- /dev/null +++ b/tools/fotff/pkg/gitee_build/gitee_build.go @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee_build + +import ( + "context" + "fotff/pkg" + "fotff/pkg/gitee_common" + "fotff/utils" + "github.com/sirupsen/logrus" + "strings" +) + +type Manager struct { + ArchiveDir string `key:"archive_dir" default:"archive"` + Workspace string `key:"workspace" default:"workspace"` + Branch string `key:"branch" default:"master"` + Component string `key:"component"` + PreCompileCMD string `key:"pre_compile_cmd"` + CompileCMD string `key:"compile_cmd"` + ImageList []string `key:"image_list"` + + *gitee_common.Manager +} + +func NewManager() pkg.Manager { + var ret Manager + utils.ParseFromConfigFile("gitee_build", &ret) + ret.Manager = gitee_common.NewManager(ret.Component, ret.Branch, ret.ArchiveDir, ret.Workspace, true) + return &ret +} + +func (m *Manager) GetNewer(cur string) (string, error) { + return m.GetNewerOrFail(cur) +} + +// Flash function implements pkg.Manager. Since this gitee_build just tests package buildings, +// there is no need to flash images actually, just build it and return nil unconditionally. +func (m *Manager) Flash(device string, pkg string, ctx context.Context) error { + logrus.Infof("now flash %s", pkg) + buildConfig := gitee_common.BuildConfig{ + Pkg: pkg, + PreCompileCMD: m.PreCompileCMD, + CompileCMD: m.CompileCMD, + ImgList: m.ImageList, + } + if m.PkgAvailable(buildConfig) { + return nil + } + if strings.Contains(buildConfig.Pkg, "build_fail") { + logrus.Warnf("here is a known build_fail token package") + } else { + if err := m.BuildNoRetry(buildConfig, true, ctx); err != nil { + logrus.Warnf("build %s fail, err: %v", pkg, err) + } else { + logrus.Infof("%s is available now", pkg) + } + } + logrus.Infof("since fotff just tests package buildings, there is no need to flash images actually, mark flash as a success unconditionally") + return nil +} diff --git a/tools/fotff/pkg/dayu200/build.go b/tools/fotff/pkg/gitee_common/build.go similarity index 55% rename from tools/fotff/pkg/dayu200/build.go rename to tools/fotff/pkg/gitee_common/build.go index 0453005..d2efd04 100644 --- a/tools/fotff/pkg/dayu200/build.go +++ b/tools/fotff/pkg/gitee_common/build.go @@ -13,7 +13,7 @@ * limitations under the License. */ -package dayu200 +package gitee_common import ( "context" @@ -25,48 +25,47 @@ import ( "path/filepath" ) -// These commands are copied from ci project. -const ( - preCompileCMD = `rm -rf prebuilts/clang/ohos/darwin-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/windows-x86_64/clang-480513;rm -rf prebuilts/clang/ohos/linux-x86_64/clang-480513;bash build/prebuilts_download.sh` - // compileCMD is copied from ci project and trim useless build-target 'make_test' to enhance build efficiency. - compileCMD = `echo 'start' && export NO_DEVTOOL=1 && export CCACHE_LOG_SUFFIX="dayu200-arm32" && export CCACHE_NOHASHDIR="true" && export CCACHE_SLOPPINESS="include_file_ctime" && ./build.sh --product-name rk3568 --ccache --build-target make_all --gn-args enable_notice_collection=false` - rmOutCMD = `rm -rf out` -) +type BuildConfig struct { + Pkg string + PreCompileCMD string + CompileCMD string + ImgList []string +} -// This list is copied from ci project. Some of them are not available, has been annotated. -var imgList = []string{ - "out/rk3568/packages/phone/images/MiniLoaderAll.bin", - "out/rk3568/packages/phone/images/boot_linux.img", - "out/rk3568/packages/phone/images/parameter.txt", - "out/rk3568/packages/phone/images/system.img", - "out/rk3568/packages/phone/images/uboot.img", - "out/rk3568/packages/phone/images/userdata.img", - "out/rk3568/packages/phone/images/vendor.img", - "out/rk3568/packages/phone/images/resource.img", - "out/rk3568/packages/phone/images/config.cfg", - "out/rk3568/packages/phone/images/ramdisk.img", - // "out/rk3568/packages/phone/images/chipset.img", - "out/rk3568/packages/phone/images/sys_prod.img", - "out/rk3568/packages/phone/images/chip_prod.img", - "out/rk3568/packages/phone/images/updater.img", - // "out/rk3568/packages/phone/updater/bin/updater_binary", +func (m *Manager) Build(config BuildConfig, ctx context.Context) error { + if m.PkgAvailable(config) { + return nil + } + logrus.Infof("%s is not available", config.Pkg) + err := m.BuildNoRetry(config, false, ctx) + if err == nil { + return nil + } + logrus.Errorf("build pkg %s err: %v", config.Pkg, err) + logrus.Infof("rm out and build pkg %s again...", config.Pkg) + err = m.BuildNoRetry(config, true, ctx) + if err == nil { + return nil + } + logrus.Errorf("build pkg %s err: %v", config.Pkg, err) + return err } -// pkgAvailable returns true if all necessary images are all available to flash. -func (m *Manager) pkgAvailable(pkg string) bool { - for _, img := range imgList { +// PkgAvailable returns true if all necessary images are all available to flash. +func (m *Manager) PkgAvailable(config BuildConfig) bool { + for _, img := range config.ImgList { imgName := filepath.Base(img) - if _, err := os.Stat(filepath.Join(m.Workspace, pkg, imgName)); err != nil { + if _, err := os.Stat(filepath.Join(m.Workspace, config.Pkg, imgName)); err != nil { return false } } return true } -// build obtain an available server, download corresponding codes, and run compile commands +// BuildNoRetry obtain an available server, download corresponding codes, and run compile commands // to build the corresponding package images, then transfer these images to the 'pkg' directory. -func (m *Manager) build(pkg string, rm bool, ctx context.Context) error { - logrus.Infof("now build %s", pkg) +func (m *Manager) BuildNoRetry(config BuildConfig, rm bool, ctx context.Context) error { + logrus.Infof("now Build %s", config.Pkg) server := res.GetBuildServer() defer res.ReleaseBuildServer(server) cmd := fmt.Sprintf("mkdir -p %s && cd %s && repo init -u https://gitee.com/openharmony/manifest.git", server.WorkSpace, server.WorkSpace) @@ -74,7 +73,7 @@ func (m *Manager) build(pkg string, rm bool, ctx context.Context) error { return fmt.Errorf("remote: mkdir error: %w", err) } if err := utils.TransFileViaSSH(utils.Upload, server.Addr, server.User, server.Passwd, - fmt.Sprintf("%s/.repo/manifest.xml", server.WorkSpace), filepath.Join(m.Workspace, pkg, "manifest_tag.xml")); err != nil { + fmt.Sprintf("%s/.repo/manifest.xml", server.WorkSpace), filepath.Join(m.Workspace, config.Pkg, "manifest_tag.xml")); err != nil { return fmt.Errorf("upload and replace manifest error: %w", err) } // 'git lfs install' may fail due to some git hooks. Call 'git lfs update --force' before install to avoid this situation. @@ -82,25 +81,25 @@ func (m *Manager) build(pkg string, rm bool, ctx context.Context) error { if err := utils.RunCmdViaSSHContext(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { return fmt.Errorf("remote: repo sync error: %w", err) } - cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, preCompileCMD) + cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, config.PreCompileCMD) if err := utils.RunCmdViaSSHContextNoRetry(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { return fmt.Errorf("remote: pre-compile command error: %w", err) } if rm { - cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, rmOutCMD) + cmd = fmt.Sprintf("cd %s && rm -rf out", server.WorkSpace) if err := utils.RunCmdViaSSHContext(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { return fmt.Errorf("remote: rm ./out command error: %w", err) } } - cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, compileCMD) + cmd = fmt.Sprintf("cd %s && %s", server.WorkSpace, config.CompileCMD) if err := utils.RunCmdViaSSHContextNoRetry(ctx, server.Addr, server.User, server.Passwd, cmd); err != nil { return fmt.Errorf("remote: compile command error: %w", err) } // has been built already, pitiful if canceled, so continue copying - for _, f := range imgList { + for _, f := range config.ImgList { imgName := filepath.Base(f) if err := utils.TransFileViaSSH(utils.Download, server.Addr, server.User, server.Passwd, - fmt.Sprintf("%s/%s", server.WorkSpace, f), filepath.Join(m.Workspace, pkg, imgName)); err != nil { + fmt.Sprintf("%s/%s", server.WorkSpace, f), filepath.Join(m.Workspace, config.Pkg, imgName)); err != nil { return fmt.Errorf("download file %s error: %w", f, err) } } diff --git a/tools/fotff/pkg/gitee_common/common.go b/tools/fotff/pkg/gitee_common/common.go new file mode 100644 index 0000000..045c1b7 --- /dev/null +++ b/tools/fotff/pkg/gitee_common/common.go @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee_common + +import ( + "code.cloudfoundry.org/archiver/extractor" + "context" + "fmt" + "fotff/utils" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "strings" + "time" +) + +type Manager struct { + Component string + Branch string + ArchiveDir string + Workspace string + WatchCI bool +} + +func NewManager(component string, branch string, archiveDir string, workspace string, watchCI bool) *Manager { + var ret = Manager{ + Component: component, + Branch: branch, + ArchiveDir: archiveDir, + Workspace: workspace, + WatchCI: watchCI, + } + go ret.cleanupOutdated() + return &ret +} + +func (m *Manager) cleanupOutdated() { + t := time.NewTicker(24 * time.Hour) + for { + <-t.C + es, err := os.ReadDir(m.Workspace) + if err != nil { + logrus.Errorf("can not read %s: %v", m.Workspace, err) + continue + } + for _, e := range es { + if !e.IsDir() { + continue + } + path := filepath.Join(m.Workspace, e.Name()) + info, err := e.Info() + if err != nil { + logrus.Errorf("can not read %s info: %v", path, err) + continue + } + if time.Now().Sub(info.ModTime()) > 7*24*time.Hour { + logrus.Warnf("%s outdated, cleanning up its contents...", path) + m.cleanupPkgFiles(path) + } + } + } +} + +func (m *Manager) cleanupPkgFiles(path string) { + es, err := os.ReadDir(path) + if err != nil { + logrus.Errorf("can not read %s: %v", path, err) + return + } + for _, e := range es { + if e.Name() == "manifest_tag.xml" || e.Name() == "__last_issue__" { + continue + } + if err := os.RemoveAll(filepath.Join(path, e.Name())); err != nil { + logrus.Errorf("remove %s fail: %v", filepath.Join(path, e.Name()), err) + } + } +} + +// Flash function implements pkg.Manager. Flash images in the 'pkg' directory to the given device. +func (m *Manager) Flash(device string, pkg string, ctx context.Context) error { + logrus.Warnf("not implemented yet") + return nil +} + +func (m *Manager) Steps(from, to string) (pkgs []string, err error) { + if from == to { + return nil, fmt.Errorf("steps err: 'from' %s and 'to' %s are the same", from, to) + } + if c, found := utils.CacheGet(fmt.Sprintf("%s_steps", m.Component), from+"__to__"+to); found { + logrus.Infof("steps from %s to %s are cached", from, to) + logrus.Infof("steps: %v", c.([]string)) + return c.([]string), nil + } + if pkgs, err = m.stepsFromGitee(from, to); err != nil { + logrus.Errorf("failed to gen steps from gitee, err: %v", err) + logrus.Warnf("fallback to getting steps from CI...") + if pkgs, err = m.stepsFromCI(from, to); err != nil { + return pkgs, err + } + return pkgs, nil + } + utils.CacheSet(fmt.Sprintf("%s_steps", m.Component), from+"__to__"+to, pkgs) + return pkgs, nil +} + +func (m *Manager) LastIssue(pkg string) (string, error) { + data, err := os.ReadFile(filepath.Join(m.Workspace, pkg, "__last_issue__")) + return string(data), err +} + +func (m *Manager) GetNewer(cur string) (string, error) { + var newFile string + if m.WatchCI { + newFile = m.getNewerFromCI(cur + ".tar.gz") + } else { + newFile = m.getNewerFileFromDir(cur+".tar.gz", func(files []os.DirEntry, i, j int) bool { + ti, _ := parseTime(files[i].Name()) + tj, _ := parseTime(files[j].Name()) + return ti.Before(tj) + }) + } + ex := extractor.NewTgz() + dirName := strings.TrimSuffix(newFile, ".tar.gz") + dir := filepath.Join(m.Workspace, dirName) + if _, err := os.Stat(dir); err == nil { + return dirName, nil + } + logrus.Infof("extracting %s to %s...", filepath.Join(m.ArchiveDir, newFile), dir) + if err := ex.Extract(filepath.Join(m.ArchiveDir, newFile), dir); err != nil { + return dirName, err + } + return dirName, nil +} + +func (m *Manager) PkgDir(pkg string) string { + return filepath.Join(m.Workspace, pkg) +} diff --git a/tools/fotff/pkg/dayu200/get_newer_ci.go b/tools/fotff/pkg/gitee_common/get_newer_ci.go similarity index 63% rename from tools/fotff/pkg/dayu200/get_newer_ci.go rename to tools/fotff/pkg/gitee_common/get_newer_ci.go index b845e2d..f5b3495 100644 --- a/tools/fotff/pkg/dayu200/get_newer_ci.go +++ b/tools/fotff/pkg/gitee_common/get_newer_ci.go @@ -13,10 +13,11 @@ * limitations under the License. */ -package dayu200 +package gitee_common import ( "encoding/json" + "fmt" "fotff/utils" "github.com/sirupsen/logrus" "io" @@ -45,22 +46,18 @@ type DailyBuildsResp struct { } type DailyBuild struct { - Id string `json:"id"` - ImgObsPath string `json:"imgObsPath"` + CurrentStatus string `json:"currentStatus"` + BuildStartTime string `json:"buildStartTime"` + BuildFailReason string `json:"buildFailReason"` + Id string `json:"id"` + ObsPath string `json:"obsPath"` + ImgObsPath string `json:"imgObsPath"` } -func (m *Manager) getNewerFromCI(cur string) string { +func (m *Manager) loopCI(param DailyBuildsQueryParam, cur string, getFn func(cur string, resp *DailyBuild) string) string { for { file := func() string { - var q = DailyBuildsQueryParam{ - ProjectName: "openharmony", - Branch: m.Branch, - Component: "dayu200", - BuildStatus: "success", - PageNum: 1, - PageSize: 1, - } - data, err := json.Marshal(q) + data, err := json.Marshal(param) if err != nil { logrus.Errorf("can not marshal query param: %v", err) return "" @@ -75,19 +72,13 @@ func (m *Manager) getNewerFromCI(cur string) string { logrus.Errorf("can not unmarshal resp [%s]: %v", string(resp), err) return "" } - if len(dailyBuildsResp.Result.DailyBuildVos) != 0 { - url := dailyBuildsResp.Result.DailyBuildVos[0].ImgObsPath - if filepath.Base(url) != cur { - logrus.Infof("new package found, name: %s", filepath.Base(url)) - file, err := m.downloadToWorkspace(url) - if err != nil { - logrus.Errorf("can not download package %s: %v", url, err) - return "" - } - return file - } + if len(dailyBuildsResp.Result.DailyBuildVos) == 0 { + return "" + } + if dailyBuildsResp.Result.DailyBuildVos[0].CurrentStatus != "end" { + return "" } - return "" + return getFn(cur, dailyBuildsResp.Result.DailyBuildVos[0]) }() if file != "" { return file @@ -96,7 +87,39 @@ func (m *Manager) getNewerFromCI(cur string) string { } } +func (m *Manager) getNewerFromCI(cur string) string { + return m.loopCI(DailyBuildsQueryParam{ + ProjectName: "openharmony", + Branch: m.Branch, + Component: m.Component, + BuildStatus: "success", + PageNum: 1, + PageSize: 1, + }, cur, m.getNewerDailyBuild) +} + +func (m *Manager) getNewerDailyBuild(cur string, db *DailyBuild) string { + p := db.ImgObsPath + if p == "" { + p = db.ObsPath + } + if filepath.Base(p) == cur { + return "" + } + logrus.Infof("new package found, name: %s", filepath.Base(p)) + file, err := m.downloadToWorkspace(p) + if err != nil { + logrus.Errorf("can not download package %s: %v", p, err) + return "" + } + return file +} + func (m *Manager) downloadToWorkspace(url string) (string, error) { + if _, err := parseTime(filepath.Base(url)); err != nil { + logrus.Errorf("can not get package time from %s, skipping", filepath.Base(url)) + return "", fmt.Errorf("can not get package time from %s, skipping", filepath.Base(url)) + } logrus.Infof("downloading %s", url) resp, err := utils.DoSimpleHttpReqRaw(http.MethodGet, url, nil, nil) if err != nil { diff --git a/tools/fotff/pkg/gitee_common/get_newer_dir.go b/tools/fotff/pkg/gitee_common/get_newer_dir.go new file mode 100644 index 0000000..03ca9a5 --- /dev/null +++ b/tools/fotff/pkg/gitee_common/get_newer_dir.go @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee_common + +import ( + "github.com/sirupsen/logrus" + "os" + "sort" + "time" +) + +func (m *Manager) getNewerFileFromDir(cur string, less func(files []os.DirEntry, i, j int) bool) string { + for { + files, err := os.ReadDir(m.ArchiveDir) + if err != nil { + logrus.Errorf("read dir %s err: %s", m.ArchiveDir, err) + time.Sleep(10 * time.Second) + continue + } + sort.Slice(files, func(i, j int) bool { + return less(files, i, j) + }) + if len(files) != 0 { + f := files[len(files)-1] + if f.Name() != cur { + logrus.Infof("new package found, name: %s", f.Name()) + return f.Name() + } + } + time.Sleep(10 * time.Second) + } +} diff --git a/tools/fotff/pkg/gitee_common/get_newer_or_fail.go b/tools/fotff/pkg/gitee_common/get_newer_or_fail.go new file mode 100644 index 0000000..03730f5 --- /dev/null +++ b/tools/fotff/pkg/gitee_common/get_newer_or_fail.go @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee_common + +import ( + "code.cloudfoundry.org/archiver/compressor" + "code.cloudfoundry.org/archiver/extractor" + "fmt" + "fotff/utils" + "github.com/sirupsen/logrus" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +func (m *Manager) GetNewerOrFail(cur string) (string, error) { + newFile := m.getNewerOrFailFromCI(cur + ".tar.gz") + ex := extractor.NewTgz() + dirName := strings.TrimSuffix(newFile, ".tar.gz") + dir := filepath.Join(m.Workspace, dirName) + if _, err := os.Stat(dir); err == nil { + return dirName, nil + } + logrus.Infof("extracting %s to %s...", filepath.Join(m.ArchiveDir, newFile), dir) + if err := ex.Extract(filepath.Join(m.ArchiveDir, newFile), dir); err != nil { + return dirName, err + } + return dirName, nil +} + +func (m *Manager) getNewerOrFailFromCI(cur string) string { + return m.loopCI(DailyBuildsQueryParam{ + ProjectName: "openharmony", + Branch: m.Branch, + Component: m.Component, + PageNum: 1, + PageSize: 1, + }, cur, m.getNewerDailyBuildOrFail) +} + +func (m *Manager) getNewerDailyBuildOrFail(cur string, db *DailyBuild) string { + switch db.BuildFailReason { + case "": + return m.getNewerDailyBuild(cur, db) + case "compile_failed": + lastSuccessTime, err := parseTime(cur) + if err != nil { + logrus.Errorf("can not get package time from %s, skipping", cur) + return "" + } + nowFailTime, err := parseTime(db.BuildStartTime) + if err != nil { + logrus.Errorf("can not get time from %s, skipping", cur) + return "" + } + if lastSuccessTime == nowFailTime { + return "" + } + return m.genFailedPackage(lastSuccessTime, nowFailTime) + default: + return "" + } +} + +func (m *Manager) genFailedPackage(lastSuccessTime, failedBuildStartTime time.Time) string { + pkg := fmt.Sprintf("%s_%s_build_fail", m.Component, failedBuildStartTime.Format("20060102_150405")) + logrus.Infof("getting failed package manifest for %s(%s) at %s", m.Component, m.Branch, failedBuildStartTime) + tags, err := m.getAllTags(lastSuccessTime, failedBuildStartTime) + if err != nil { + logrus.Errorf("can not get latest tag from ci, err: %v", err) + return "" + } + if len(tags) == 0 { + logrus.Error("can not get latest tag from ci, tag list is empty") + return "" + } + if err := os.MkdirAll(filepath.Join(m.Workspace, pkg), 0750); err != nil { + logrus.Errorf("can not mkdir %s, err: %v", pkg, err) + return "" + } + resp, err := utils.DoSimpleHttpReq(http.MethodGet, tags[len(tags)-1].TagFileURL, nil, nil) + if err != nil { + logrus.Errorf("can not get latest tag file from ci, err: %v", err) + return "" + } + err = os.WriteFile(filepath.Join(m.Workspace, pkg, "manifest_tag.xml"), resp, 0640) + if err != nil { + logrus.Errorf("can not write latest tag file, err: %v", err) + return "" + } + if err := compressor.NewTgz().Compress(filepath.Join(m.Workspace, pkg), filepath.Join(m.ArchiveDir, pkg+".tar.gz")); err != nil { + logrus.Errorf("can not write latest tag file, err: %v", err) + return "" + } + return pkg + ".tar.gz" +} diff --git a/tools/fotff/pkg/dayu200/steps_ci.go b/tools/fotff/pkg/gitee_common/steps_ci.go similarity index 95% rename from tools/fotff/pkg/dayu200/steps_ci.go rename to tools/fotff/pkg/gitee_common/steps_ci.go index 577f9f2..36cdd9e 100644 --- a/tools/fotff/pkg/dayu200/steps_ci.go +++ b/tools/fotff/pkg/gitee_common/steps_ci.go @@ -13,7 +13,7 @@ * limitations under the License. */ -package dayu200 +package gitee_common import ( "encoding/json" @@ -53,11 +53,11 @@ type Tag struct { } func (m *Manager) stepsFromCI(from, to string) (pkgs []string, err error) { - startTime, err := getPackageTime(from) + startTime, err := parseTime(from) if err != nil { return nil, err } - endTime, err := getPackageTime(to) + endTime, err := parseTime(to) if err != nil { return nil, err } @@ -69,9 +69,6 @@ func (m *Manager) getAllStepsFromTags(from, to time.Time) (pkgs []string, err er if err != nil { return nil, err } - sort.Slice(tags, func(i, j int) bool { - return tags[i].Timestamp < tags[j].Timestamp - }) for _, tag := range tags { pkg, err := m.genTagPackage(tag) if err != nil { @@ -125,6 +122,9 @@ func (m *Manager) getAllTags(from, to time.Time) (ret []*Tag, err error) { } pageNum++ } + sort.Slice(ret, func(i, j int) bool { + return ret[i].Timestamp < ret[j].Timestamp + }) return ret, nil } diff --git a/tools/fotff/pkg/dayu200/steps_gitee.go b/tools/fotff/pkg/gitee_common/steps_gitee.go similarity index 94% rename from tools/fotff/pkg/dayu200/steps_gitee.go rename to tools/fotff/pkg/gitee_common/steps_gitee.go index 543c82a..11899fd 100644 --- a/tools/fotff/pkg/dayu200/steps_gitee.go +++ b/tools/fotff/pkg/gitee_common/steps_gitee.go @@ -1,365 +1,367 @@ -/* - * Copyright (c) 2022 Huawei Device Co., Ltd. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dayu200 - -import ( - "bufio" - "bytes" - "encoding/xml" - "fmt" - "fotff/vcs" - "fotff/vcs/gitee" - "github.com/huandu/go-clone" - "github.com/sirupsen/logrus" - "os" - "path/filepath" - "regexp" - "sort" - "strconv" - "strings" - "sync" - "time" -) - -type IssueInfo struct { - visited bool - RelatedIssues []string - MRs []*gitee.Commit - StructCTime string - StructureUpdates []*vcs.ProjectUpdate -} - -type Step struct { - IssueURLs []string - MRs []*gitee.Commit - StructCTime string - StructureUpdates []*vcs.ProjectUpdate -} - -func (m *Manager) stepsFromGitee(from, to string) (pkgs []string, err error) { - updates, err := m.getRepoUpdates(from, to) - if err != nil { - return nil, err - } - startTime, err := getPackageTime(from) - if err != nil { - return nil, err - } - endTime, err := getPackageTime(to) - if err != nil { - return nil, err - } - logrus.Infof("find %d repo updates from %s to %s", len(updates), from, to) - steps, err := getAllStepsFromGitee(startTime, endTime, m.Branch, updates) - if err != nil { - return nil, err - } - logrus.Infof("find total %d steps from %s to %s", len(steps), from, to) - baseManifest, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml")) - if err != nil { - return nil, err - } - for _, step := range steps { - var newPkg string - if newPkg, baseManifest, err = m.genStepPackage(baseManifest, step); err != nil { - return nil, err - } - pkgs = append(pkgs, newPkg) - } - return pkgs, nil -} - -func (m *Manager) getRepoUpdates(from, to string) (updates []vcs.ProjectUpdate, err error) { - m1, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml")) - if err != nil { - return nil, err - } - m2, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, to, "manifest_tag.xml")) - if err != nil { - return nil, err - } - return vcs.GetRepoUpdates(m1, m2) -} - -func getAllStepsFromGitee(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (ret []Step, err error) { - allMRs, err := getAllMRs(startTime, endTime, branch, updates) - if err != nil { - return nil, err - } - issueInfos, err := combineMRsToIssue(allMRs, branch) - if err != nil { - return nil, err - } - return combineIssuesToStep(issueInfos) -} - -func getAllMRs(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (allMRs []*gitee.Commit, err error) { - var once sync.Once - for _, update := range updates { - var prs []*gitee.Commit - if update.P1.StructureDiff(update.P2) { - once.Do(func() { - prs, err = gitee.GetBetweenTimeMRs("openharmony", "manifest", branch, startTime, endTime) - }) - if update.P1 != nil { - var p1 []*gitee.Commit - p1, err = gitee.GetBetweenTimeMRs("openharmony", update.P1.Name, branch, startTime, endTime) - prs = append(prs, p1...) - } - if update.P2 != nil { - var p2 []*gitee.Commit - p2, err = gitee.GetBetweenTimeMRs("openharmony", update.P2.Name, branch, startTime, endTime) - prs = append(prs, p2...) - } - } else { - prs, err = gitee.GetBetweenMRs(gitee.CompareParam{ - Head: update.P2.Revision, - Base: update.P1.Revision, - Owner: "openharmony", - Repo: update.P2.Name, - }) - } - if err != nil { - return nil, err - } - allMRs = append(allMRs, prs...) - } - logrus.Infof("find total %d merge request commits of all repo updates", len(allMRs)) - return -} - -func combineMRsToIssue(allMRs []*gitee.Commit, branch string) (map[string]*IssueInfo, error) { - ret := make(map[string]*IssueInfo) - for _, mr := range allMRs { - num, err := strconv.Atoi(strings.Trim(regexp.MustCompile(`!\d+ `).FindString(mr.Commit.Message), "! ")) - if err != nil { - return nil, fmt.Errorf("parse MR message for %s fail: %s", mr.URL, err) - } - issues, err := gitee.GetMRIssueURL(mr.Owner, mr.Repo, num) - if err != nil { - return nil, err - } - if len(issues) == 0 { - issues = []string{mr.URL} - } - var scs []*vcs.ProjectUpdate - var scTime string - if mr.Owner == "openharmony" && mr.Repo == "manifest" { - if scTime, scs, err = parseStructureUpdates(mr, branch); err != nil { - return nil, err - } - } - for i, issue := range issues { - if _, ok := ret[issue]; !ok { - ret[issue] = &IssueInfo{ - MRs: []*gitee.Commit{mr}, - RelatedIssues: append(issues[:i], issues[i+1:]...), - StructCTime: scTime, - StructureUpdates: scs, - } - } else { - ret[issue] = &IssueInfo{ - MRs: append(ret[issue].MRs, mr), - RelatedIssues: append(ret[issue].RelatedIssues, append(issues[:i], issues[i+1:]...)...), - StructCTime: scTime, - StructureUpdates: append(ret[issue].StructureUpdates, scs...), - } - } - } - } - logrus.Infof("find total %d issues of all repo updates", len(ret)) - return ret, nil -} - -func combineOtherRelatedIssue(parent, self *IssueInfo, all map[string]*IssueInfo) { - if self.visited { - return - } - self.visited = true - for _, other := range self.RelatedIssues { - if son, ok := all[other]; ok { - combineOtherRelatedIssue(self, son, all) - delete(all, other) - } - } - parent.RelatedIssues = deDupIssues(append(parent.RelatedIssues, self.RelatedIssues...)) - parent.MRs = deDupMRs(append(parent.MRs, self.MRs...)) - parent.StructureUpdates = deDupProjectUpdates(append(parent.StructureUpdates, self.StructureUpdates...)) - if len(parent.StructCTime) != 0 && parent.StructCTime < self.StructCTime { - parent.StructCTime = self.StructCTime - } -} - -func deDupProjectUpdates(us []*vcs.ProjectUpdate) (retMRs []*vcs.ProjectUpdate) { - dupIndexes := make([]bool, len(us)) - for i := range us { - for j := i + 1; j < len(us); j++ { - if us[j].P1 == us[i].P1 && us[j].P2 == us[i].P2 { - dupIndexes[j] = true - } - } - } - for i, dup := range dupIndexes { - if dup { - continue - } - retMRs = append(retMRs, us[i]) - } - return -} - -func deDupMRs(mrs []*gitee.Commit) (retMRs []*gitee.Commit) { - tmp := make(map[string]*gitee.Commit) - for _, m := range mrs { - tmp[m.SHA] = m - } - for _, m := range tmp { - retMRs = append(retMRs, m) - } - return -} - -func deDupIssues(issues []string) (retIssues []string) { - tmp := make(map[string]string) - for _, i := range issues { - tmp[i] = i - } - for _, i := range tmp { - retIssues = append(retIssues, i) - } - return -} - -// parseStructureUpdates get changed XMLs and parse it to recognize repo structure changes. -// Since we do not care which revision a repo was, P1 is not welly handled, just assign it not nil for performance. -func parseStructureUpdates(commit *gitee.Commit, branch string) (string, []*vcs.ProjectUpdate, error) { - tmp := make(map[string]vcs.ProjectUpdate) - if len(commit.Files) == 0 { - // commit that queried from MR req does not contain file details, should fetch again - var err error - if commit, err = gitee.GetCommit(commit.Owner, commit.Repo, commit.SHA); err != nil { - return "", nil, err - } - } - for _, f := range commit.Files { - if filepath.Ext(f.Filename) != ".xml" { - continue - } - if err := parseFilePatch(f.Patch, tmp); err != nil { - return "", nil, err - } - } - var ret []*vcs.ProjectUpdate - for _, pu := range tmp { - projectUpdateCopy := pu - ret = append(ret, &projectUpdateCopy) - } - for _, pu := range ret { - if pu.P1 == nil && pu.P2 != nil { - lastCommit, err := gitee.GetLatestMRBefore("openharmony", pu.P2.Name, branch, commit.Commit.Committer.Date) - if err != nil { - return "", nil, err - } - pu.P2.Revision = lastCommit.SHA - } - } - return commit.Commit.Committer.Date, ret, nil -} - -func parseFilePatch(str string, m map[string]vcs.ProjectUpdate) error { - sc := bufio.NewScanner(bytes.NewBuffer([]byte(str))) - for sc.Scan() { - line := sc.Text() - var p vcs.Project - if strings.HasPrefix(line, "-") { - if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil { - m[p.Name] = vcs.ProjectUpdate{P1: &p, P2: m[p.Name].P2} - } - } else if strings.HasPrefix(line, "+") { - if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil { - m[p.Name] = vcs.ProjectUpdate{P1: m[p.Name].P1, P2: &p} - } - } - } - return nil -} - -func combineIssuesToStep(issueInfos map[string]*IssueInfo) (ret []Step, err error) { - for _, info := range issueInfos { - combineOtherRelatedIssue(info, info, issueInfos) - } - for issue, infos := range issueInfos { - sort.Slice(infos.MRs, func(i, j int) bool { - // move the latest MR to the first place, use its merged_time to represent the update time of the issue - return infos.MRs[i].Commit.Committer.Date > infos.MRs[j].Commit.Committer.Date - }) - ret = append(ret, Step{ - IssueURLs: append(infos.RelatedIssues, issue), - MRs: infos.MRs, - StructCTime: infos.StructCTime, - StructureUpdates: infos.StructureUpdates}) - } - sort.Slice(ret, func(i, j int) bool { - ti, tj := ret[i].MRs[0].Commit.Committer.Date, ret[j].MRs[0].Commit.Committer.Date - if len(ret[i].StructCTime) != 0 { - ti = ret[i].StructCTime - } - if len(ret[j].StructCTime) != 0 { - ti = ret[j].StructCTime - } - return ti < tj - }) - logrus.Infof("find total %d steps of all issues", len(ret)) - return -} - -var simpleRegTimeInPkgName = regexp.MustCompile(`\d{8}_\d{6}`) - -func getPackageTime(pkg string) (time.Time, error) { - return time.ParseInLocation(`20060102_150405`, simpleRegTimeInPkgName.FindString(pkg), time.Local) -} - -func (m *Manager) genStepPackage(base *vcs.Manifest, step Step) (newPkg string, newManifest *vcs.Manifest, err error) { - defer func() { - logrus.Infof("package dir %s for step %v generated", newPkg, step.IssueURLs) - }() - newManifest = clone.Clone(base).(*vcs.Manifest) - for _, u := range step.StructureUpdates { - if u.P2 != nil { - newManifest.UpdateManifestProject(u.P2.Name, u.P2.Path, u.P2.Remote, u.P2.Revision, true) - } else if u.P1 != nil { - newManifest.RemoveManifestProject(u.P1.Name) - } - } - for _, mr := range step.MRs { - newManifest.UpdateManifestProject(mr.Repo, "", "", mr.SHA, false) - } - md5sum, err := newManifest.Standardize() - if err != nil { - return "", nil, err - } - if err := os.MkdirAll(filepath.Join(m.Workspace, md5sum), 0750); err != nil { - return "", nil, err - } - if err := os.WriteFile(filepath.Join(m.Workspace, md5sum, "__last_issue__"), []byte(fmt.Sprintf("%v", step.IssueURLs)), 0640); err != nil { - return "", nil, err - } - err = newManifest.WriteFile(filepath.Join(m.Workspace, md5sum, "manifest_tag.xml")) - if err != nil { - return "", nil, err - } - return md5sum, newManifest, nil -} +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package gitee_common + +import ( + "bufio" + "bytes" + "encoding/xml" + "fmt" + "fotff/vcs" + "fotff/vcs/gitee" + "github.com/huandu/go-clone" + "github.com/sirupsen/logrus" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +type IssueInfo struct { + visited bool + RelatedIssues []string + MRs []*gitee.Commit + StructCTime string + StructureUpdates []*vcs.ProjectUpdate +} + +type Step struct { + IssueURLs []string + MRs []*gitee.Commit + StructCTime string + StructureUpdates []*vcs.ProjectUpdate +} + +func (m *Manager) stepsFromGitee(from, to string) (pkgs []string, err error) { + updates, err := m.getRepoUpdates(from, to) + if err != nil { + return nil, err + } + startTime, err := parseTime(from) + if err != nil { + return nil, err + } + endTime, err := parseTime(to) + if err != nil { + return nil, err + } + logrus.Infof("find %d repo updates from %s to %s", len(updates), from, to) + steps, err := getAllStepsFromGitee(startTime, endTime, m.Branch, updates) + if err != nil { + return nil, err + } + logrus.Infof("find total %d steps from %s to %s", len(steps), from, to) + baseManifest, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml")) + if err != nil { + return nil, err + } + for _, step := range steps { + var newPkg string + if newPkg, baseManifest, err = m.genStepPackage(baseManifest, step); err != nil { + return nil, err + } + pkgs = append(pkgs, newPkg) + } + return pkgs, nil +} + +func (m *Manager) getRepoUpdates(from, to string) (updates []vcs.ProjectUpdate, err error) { + m1, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, from, "manifest_tag.xml")) + if err != nil { + return nil, err + } + m2, err := vcs.ParseManifestFile(filepath.Join(m.Workspace, to, "manifest_tag.xml")) + if err != nil { + return nil, err + } + return vcs.GetRepoUpdates(m1, m2) +} + +func getAllStepsFromGitee(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (ret []Step, err error) { + allMRs, err := getAllMRs(startTime, endTime, branch, updates) + if err != nil { + return nil, err + } + issueInfos, err := combineMRsToIssue(allMRs, branch) + if err != nil { + return nil, err + } + return combineIssuesToStep(issueInfos) +} + +func getAllMRs(startTime, endTime time.Time, branch string, updates []vcs.ProjectUpdate) (allMRs []*gitee.Commit, err error) { + var once sync.Once + for _, update := range updates { + var prs []*gitee.Commit + if update.P1.StructureDiff(update.P2) { + once.Do(func() { + prs, err = gitee.GetBetweenTimeMRs("openharmony", "manifest", branch, startTime, endTime) + }) + if update.P1 != nil { + var p1 []*gitee.Commit + p1, err = gitee.GetBetweenTimeMRs("openharmony", update.P1.Name, branch, startTime, endTime) + prs = append(prs, p1...) + } + if update.P2 != nil { + var p2 []*gitee.Commit + p2, err = gitee.GetBetweenTimeMRs("openharmony", update.P2.Name, branch, startTime, endTime) + prs = append(prs, p2...) + } + } else { + prs, err = gitee.GetBetweenMRs(gitee.CompareParam{ + Head: update.P2.Revision, + Base: update.P1.Revision, + Owner: "openharmony", + Repo: update.P2.Name, + }) + } + if err != nil { + return nil, err + } + allMRs = append(allMRs, prs...) + } + logrus.Infof("find total %d merge request commits of all repo updates", len(allMRs)) + return +} + +func combineMRsToIssue(allMRs []*gitee.Commit, branch string) (map[string]*IssueInfo, error) { + ret := make(map[string]*IssueInfo) + for _, mr := range allMRs { + num, err := strconv.Atoi(strings.Trim(regexp.MustCompile(`!\d+ `).FindString(mr.Commit.Message), "! ")) + if err != nil { + return nil, fmt.Errorf("parse MR message for %s fail: %s", mr.URL, err) + } + issues, err := gitee.GetMRIssueURL(mr.Owner, mr.Repo, num) + if err != nil { + return nil, err + } + if len(issues) == 0 { + issues = []string{mr.URL} + } + var scs []*vcs.ProjectUpdate + var scTime string + if mr.Owner == "openharmony" && mr.Repo == "manifest" { + if scTime, scs, err = parseStructureUpdates(mr, branch); err != nil { + return nil, err + } + } + for i, issue := range issues { + if _, ok := ret[issue]; !ok { + ret[issue] = &IssueInfo{ + MRs: []*gitee.Commit{mr}, + RelatedIssues: append(issues[:i], issues[i+1:]...), + StructCTime: scTime, + StructureUpdates: scs, + } + } else { + ret[issue] = &IssueInfo{ + MRs: append(ret[issue].MRs, mr), + RelatedIssues: append(ret[issue].RelatedIssues, append(issues[:i], issues[i+1:]...)...), + StructCTime: scTime, + StructureUpdates: append(ret[issue].StructureUpdates, scs...), + } + } + } + } + logrus.Infof("find total %d issues of all repo updates", len(ret)) + return ret, nil +} + +func combineOtherRelatedIssue(parent, self *IssueInfo, all map[string]*IssueInfo) { + if self.visited { + return + } + self.visited = true + for _, other := range self.RelatedIssues { + if son, ok := all[other]; ok { + combineOtherRelatedIssue(self, son, all) + delete(all, other) + } + } + parent.RelatedIssues = deDupIssues(append(parent.RelatedIssues, self.RelatedIssues...)) + parent.MRs = deDupMRs(append(parent.MRs, self.MRs...)) + parent.StructureUpdates = deDupProjectUpdates(append(parent.StructureUpdates, self.StructureUpdates...)) + if len(parent.StructCTime) != 0 && parent.StructCTime < self.StructCTime { + parent.StructCTime = self.StructCTime + } +} + +func deDupProjectUpdates(us []*vcs.ProjectUpdate) (retMRs []*vcs.ProjectUpdate) { + dupIndexes := make([]bool, len(us)) + for i := range us { + for j := i + 1; j < len(us); j++ { + if us[j].P1 == us[i].P1 && us[j].P2 == us[i].P2 { + dupIndexes[j] = true + } + } + } + for i, dup := range dupIndexes { + if dup { + continue + } + retMRs = append(retMRs, us[i]) + } + return +} + +func deDupMRs(mrs []*gitee.Commit) (retMRs []*gitee.Commit) { + tmp := make(map[string]*gitee.Commit) + for _, m := range mrs { + tmp[m.SHA] = m + } + for _, m := range tmp { + retMRs = append(retMRs, m) + } + return +} + +func deDupIssues(issues []string) (retIssues []string) { + tmp := make(map[string]string) + for _, i := range issues { + tmp[i] = i + } + for _, i := range tmp { + retIssues = append(retIssues, i) + } + return +} + +// parseStructureUpdates get changed XMLs and parse it to recognize repo structure changes. +// Since we do not care which revision a repo was, P1 is not welly handled, just assign it not nil for performance. +func parseStructureUpdates(commit *gitee.Commit, branch string) (string, []*vcs.ProjectUpdate, error) { + tmp := make(map[string]vcs.ProjectUpdate) + if len(commit.Files) == 0 { + // commit that queried from MR req does not contain file details, should fetch again + var err error + if commit, err = gitee.GetCommit(commit.Owner, commit.Repo, commit.SHA); err != nil { + return "", nil, err + } + } + for _, f := range commit.Files { + if filepath.Ext(f.Filename) != ".xml" { + continue + } + if err := parseFilePatch(f.Patch, tmp); err != nil { + return "", nil, err + } + } + var ret []*vcs.ProjectUpdate + for _, pu := range tmp { + projectUpdateCopy := pu + ret = append(ret, &projectUpdateCopy) + } + for _, pu := range ret { + if pu.P1 == nil && pu.P2 != nil { + lastCommit, err := gitee.GetLatestMRBefore("openharmony", pu.P2.Name, branch, commit.Commit.Committer.Date) + if err != nil { + return "", nil, err + } + pu.P2.Revision = lastCommit.SHA + } + } + return commit.Commit.Committer.Date, ret, nil +} + +func parseFilePatch(str string, m map[string]vcs.ProjectUpdate) error { + sc := bufio.NewScanner(bytes.NewBuffer([]byte(str))) + for sc.Scan() { + line := sc.Text() + var p vcs.Project + if strings.HasPrefix(line, "-") { + if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil { + m[p.Name] = vcs.ProjectUpdate{P1: &p, P2: m[p.Name].P2} + } + } else if strings.HasPrefix(line, "+") { + if err := xml.Unmarshal([]byte(line[1:]), &p); err == nil { + m[p.Name] = vcs.ProjectUpdate{P1: m[p.Name].P1, P2: &p} + } + } + } + return nil +} + +func combineIssuesToStep(issueInfos map[string]*IssueInfo) (ret []Step, err error) { + for _, info := range issueInfos { + combineOtherRelatedIssue(info, info, issueInfos) + } + for issue, infos := range issueInfos { + sort.Slice(infos.MRs, func(i, j int) bool { + // move the latest MR to the first place, use its merged_time to represent the update time of the issue + return infos.MRs[i].Commit.Committer.Date > infos.MRs[j].Commit.Committer.Date + }) + ret = append(ret, Step{ + IssueURLs: append(infos.RelatedIssues, issue), + MRs: infos.MRs, + StructCTime: infos.StructCTime, + StructureUpdates: infos.StructureUpdates}) + } + sort.Slice(ret, func(i, j int) bool { + ti, tj := ret[i].MRs[0].Commit.Committer.Date, ret[j].MRs[0].Commit.Committer.Date + if len(ret[i].StructCTime) != 0 { + ti = ret[i].StructCTime + } + if len(ret[j].StructCTime) != 0 { + ti = ret[j].StructCTime + } + return ti < tj + }) + logrus.Infof("find total %d steps of all issues", len(ret)) + return +} + +func parseTime(pkg string) (time.Time, error) { + t, err := time.ParseInLocation(`20060102_150405`, regexp.MustCompile(`\d{8}_\d{6}`).FindString(pkg), time.Local) + if err != nil { + return time.ParseInLocation(`20060102150405`, regexp.MustCompile(`\d{14}`).FindString(pkg), time.Local) + } + return t, nil +} + +func (m *Manager) genStepPackage(base *vcs.Manifest, step Step) (newPkg string, newManifest *vcs.Manifest, err error) { + defer func() { + logrus.Infof("package dir %s for step %v generated", newPkg, step.IssueURLs) + }() + newManifest = clone.Clone(base).(*vcs.Manifest) + for _, u := range step.StructureUpdates { + if u.P2 != nil { + newManifest.UpdateManifestProject(u.P2.Name, u.P2.Path, u.P2.Remote, u.P2.Revision, true) + } else if u.P1 != nil { + newManifest.RemoveManifestProject(u.P1.Name) + } + } + for _, mr := range step.MRs { + newManifest.UpdateManifestProject(mr.Repo, "", "", mr.SHA, false) + } + md5sum, err := newManifest.Standardize() + if err != nil { + return "", nil, err + } + if err := os.MkdirAll(filepath.Join(m.Workspace, md5sum), 0750); err != nil { + return "", nil, err + } + if err := os.WriteFile(filepath.Join(m.Workspace, md5sum, "__last_issue__"), []byte(fmt.Sprintf("%v", step.IssueURLs)), 0640); err != nil { + return "", nil, err + } + err = newManifest.WriteFile(filepath.Join(m.Workspace, md5sum, "manifest_tag.xml")) + if err != nil { + return "", nil, err + } + return md5sum, newManifest, nil +} diff --git a/tools/fotff/pkg/dayu200/steps_gitee_test.go b/tools/fotff/pkg/gitee_common/steps_gitee_test.go similarity index 99% rename from tools/fotff/pkg/dayu200/steps_gitee_test.go rename to tools/fotff/pkg/gitee_common/steps_gitee_test.go index f7b86f1..38f8dcb 100644 --- a/tools/fotff/pkg/dayu200/steps_gitee_test.go +++ b/tools/fotff/pkg/gitee_common/steps_gitee_test.go @@ -13,7 +13,7 @@ * limitations under the License. */ -package dayu200 +package gitee_common import ( "fotff/vcs" diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_080109-dayu200/manifest_tag.xml b/tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221201_080109-dayu200/manifest_tag.xml similarity index 100% rename from tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_080109-dayu200/manifest_tag.xml rename to tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221201_080109-dayu200/manifest_tag.xml diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_100141-dayu200/manifest_tag.xml b/tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221201_100141-dayu200/manifest_tag.xml similarity index 100% rename from tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221201_100141-dayu200/manifest_tag.xml rename to tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221201_100141-dayu200/manifest_tag.xml diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_110027-dayu200/manifest_tag.xml b/tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221213_110027-dayu200/manifest_tag.xml similarity index 100% rename from tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_110027-dayu200/manifest_tag.xml rename to tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221213_110027-dayu200/manifest_tag.xml diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_140150-dayu200/manifest_tag.xml b/tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221213_140150-dayu200/manifest_tag.xml similarity index 100% rename from tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221213_140150-dayu200/manifest_tag.xml rename to tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221213_140150-dayu200/manifest_tag.xml diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_100124-dayu200/manifest_tag.xml b/tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221214_100124-dayu200/manifest_tag.xml similarity index 100% rename from tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_100124-dayu200/manifest_tag.xml rename to tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221214_100124-dayu200/manifest_tag.xml diff --git a/tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_110125-dayu200/manifest_tag.xml b/tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221214_110125-dayu200/manifest_tag.xml similarity index 100% rename from tools/fotff/pkg/dayu200/testdata/version-Daily_Version-dayu200-20221214_110125-dayu200/manifest_tag.xml rename to tools/fotff/pkg/gitee_common/testdata/version-Daily_Version-dayu200-20221214_110125-dayu200/manifest_tag.xml diff --git a/tools/fotff/pkg/pkg.go b/tools/fotff/pkg/pkg.go index 142ae30..0e434b4 100644 --- a/tools/fotff/pkg/pkg.go +++ b/tools/fotff/pkg/pkg.go @@ -17,10 +17,6 @@ package pkg import ( "context" - "github.com/sirupsen/logrus" - "os" - "sort" - "time" ) type NewFunc func() Manager @@ -37,25 +33,3 @@ type Manager interface { // PkgDir returns where pkg exists in the filesystem. PkgDir(pkg string) string } - -func GetNewerFileFromDir(dir string, cur string, less func(files []os.DirEntry, i, j int) bool) string { - for { - files, err := os.ReadDir(dir) - if err != nil { - logrus.Errorf("read dir %s err: %s", dir, err) - time.Sleep(10 * time.Second) - continue - } - sort.Slice(files, func(i, j int) bool { - return less(files, i, j) - }) - if len(files) != 0 { - f := files[len(files)-1] - if f.Name() != cur { - logrus.Infof("new package found, name: %s", f.Name()) - return f.Name() - } - } - time.Sleep(10 * time.Second) - } -} diff --git a/tools/fotff/rec/fotff.go b/tools/fotff/rec/fotff.go index 84e108f..64cccd8 100644 --- a/tools/fotff/rec/fotff.go +++ b/tools/fotff/rec/fotff.go @@ -146,18 +146,7 @@ func flashAndTest(m pkg.Manager, t tester.Tester, pkg string, testcase string, c device := res.GetDevice() defer res.ReleaseDevice(device) if err := m.Flash(device, pkg, ctx); err != nil && !errors.Is(err, context.Canceled) { - // Sometimes we need to find out the first compilation failure. Treat it as a normal test failure to re-use this framework. - var cfg struct { - AllowBuildError string `key:"allow_build_err"` - } - utils.ParseFromConfigFile("", &cfg) - if cfg.AllowBuildError != "true" { - return false, newFellows, err - } - logrus.Warnf("can not flash %s to %s, assume it as a failure: %v", pkg, device, err) - for _, cases := range append(fellows, testcase) { - results = append(results, tester.Result{TestCaseName: cases, Status: tester.ResultFail}) - } + return false, newFellows, err } else { if err = t.Prepare(m.PkgDir(pkg), device, ctx); err != nil { return false, newFellows, err diff --git a/tools/fotff/tester/pkg_available/pkg_available.go b/tools/fotff/tester/pkg_available/pkg_available.go new file mode 100644 index 0000000..602d550 --- /dev/null +++ b/tools/fotff/tester/pkg_available/pkg_available.go @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pkg_available + +import ( + "context" + "fotff/tester" + "github.com/sirupsen/logrus" + "math/rand" + "os" + "strings" + "sync" + "time" +) + +type Tester struct { + device2PkgDir sync.Map +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func NewTester() tester.Tester { + ret := &Tester{} + return ret +} + +func (t *Tester) TaskName() string { + return "pkg_available" +} + +func (t *Tester) Prepare(pkgDir string, device string, ctx context.Context) error { + t.device2PkgDir.Store(device, pkgDir) + return nil +} + +func (t *Tester) DoTestTask(deviceSN string, ctx context.Context) (ret []tester.Result, err error) { + return t.DoTestCases(deviceSN, []string{"pkg_available"}, ctx) +} + +func (t *Tester) DoTestCase(deviceSN, testCase string, ctx context.Context) (ret tester.Result, err error) { + pkgDir, _ := t.device2PkgDir.Load(deviceSN) + es, err := os.ReadDir(pkgDir.(string)) + if err != nil { + logrus.Errorf("can not read dir %s, testcase failed", pkgDir.(string)) + return tester.Result{TestCaseName: testCase, Status: tester.ResultFail}, nil + } + for _, e := range es { + if strings.HasSuffix(e.Name(), ".img") { + logrus.Infof("find image in dir %s, package is avaliable, testcase pass", pkgDir.(string)) + return tester.Result{TestCaseName: testCase, Status: tester.ResultPass}, nil + } + } + logrus.Infof("no images in dir %s, package is not avaliable, testcase failed", pkgDir.(string)) + return tester.Result{TestCaseName: testCase, Status: tester.ResultFail}, nil +} + +func (t *Tester) DoTestCases(deviceSN string, testcases []string, ctx context.Context) (ret []tester.Result, err error) { + for _, testcase := range testcases { + r, err := t.DoTestCase(deviceSN, testcase, ctx) + if err != nil { + return nil, err + } + ret = append(ret, r) + } + return ret, nil +} diff --git a/tools/fotff/utils/http.go b/tools/fotff/utils/http.go index e2ed194..4b4bfcb 100644 --- a/tools/fotff/utils/http.go +++ b/tools/fotff/utils/http.go @@ -25,7 +25,11 @@ import ( ) func DoSimpleHttpReqRaw(method string, url string, body []byte, header map[string]string) (response *http.Response, err error) { - for i := 0; i < 3; i++ { + maxRetry := len(proxyList) + if maxRetry < 3 { + maxRetry = 3 + } + for i := 0; i < maxRetry; i++ { if response, err = doSimpleHttpReqImpl(method, url, body, header); err == nil { return } @@ -36,7 +40,11 @@ func DoSimpleHttpReqRaw(method string, url string, body []byte, header map[strin func DoSimpleHttpReq(method string, url string, body []byte, header map[string]string) (ret []byte, err error) { var resp *http.Response - for i := 0; i < 3; i++ { + maxRetry := len(proxyList) + if maxRetry < 3 { + maxRetry = 3 + } + for i := 0; i < maxRetry; i++ { if resp, err = doSimpleHttpReqImpl(method, url, body, header); err == nil { ret, err = io.ReadAll(resp.Body) resp.Body.Close() diff --git a/tools/fotff/utils/ini.go b/tools/fotff/utils/ini.go index 2680e7b..b0d5e67 100644 --- a/tools/fotff/utils/ini.go +++ b/tools/fotff/utils/ini.go @@ -19,6 +19,7 @@ import ( "github.com/Unknwon/goconfig" "github.com/sirupsen/logrus" "reflect" + "strings" ) // ParseFromConfigFile parse ini file and set values by the tag of fields. @@ -46,6 +47,22 @@ func ParseFromConfigFile(section string, p any) { v = rt.Elem().Field(i).Tag.Get("default") } rv.Elem().Field(i).SetString(v) + case reflect.Slice: + if rt.Elem().Field(i).Type.Elem().Kind() != reflect.String { + break + } + key := rt.Elem().Field(i).Tag.Get("key") + if key == "" { + continue + } + var v string + if conf != nil { + v, err = conf.GetValue(section, key) + } + if conf == nil || err != nil { + v = rt.Elem().Field(i).Tag.Get("default") + } + rv.Elem().Field(i).Set(reflect.ValueOf(strings.Split(v, ","))) case reflect.Struct: ParseFromConfigFile(section, rv.Elem().Field(i).Addr().Interface()) } -- Gitee