diff --git a/.drone.yml b/.drone.yml index 459536621b8..4e7789ef923 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,749 +1,3 @@ ---- -kind: pipeline -type: docker -name: compliance - -platform: - os: linux - arch: amd64 - -trigger: - event: - - pull_request - paths: - exclude: - - "docs/**" - - "*.md" - -volumes: - - name: deps - temp: {} - -steps: - - name: deps-frontend - image: node:18 - pull: always - commands: - - make deps-frontend - - - name: deps-backend - image: gitea/test_env:linux-1.20-amd64 - pull: always - commands: - - make deps-backend - - make deps-tools - volumes: - - name: deps - path: /go - - - name: lint-frontend - image: node:18 - commands: - - make lint-frontend - depends_on: [deps-frontend] - - - name: lint-backend - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - pull: always - commands: - - make lint-backend - environment: - GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not - GOSUMDB: sum.golang.org - TAGS: bindata sqlite sqlite_unlock_notify - depends_on: [deps-backend] - volumes: - - name: deps - path: /go - - - name: lint-backend-windows - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - commands: - - make lint-go-windows lint-go-vet - environment: - GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not - GOSUMDB: sum.golang.org - TAGS: bindata sqlite sqlite_unlock_notify - GOOS: windows - GOARCH: amd64 - depends_on: [deps-backend] - volumes: - - name: deps - path: /go - - - name: lint-backend-gogit - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - commands: - - make lint-backend - environment: - GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not - GOSUMDB: sum.golang.org - TAGS: bindata gogit sqlite sqlite_unlock_notify - depends_on: [deps-backend] - volumes: - - name: deps - path: /go - - - name: checks-frontend - image: node:18 - commands: - - make checks-frontend - depends_on: [deps-frontend] - - - name: checks-backend - image: gitea/test_env:linux-1.20-amd64 - commands: - - make --always-make checks-backend # ensure the 'go-licenses' make target runs - depends_on: [deps-backend] - volumes: - - name: deps - path: /go - - - name: test-frontend - image: node:18 - commands: - - make test-frontend - depends_on: [lint-frontend] - - - name: build-frontend - image: node:18 - commands: - - make frontend - depends_on: [deps-frontend] - - - name: build-backend-no-gcc - image: gitea/test_env:linux-1.19-amd64 # this step is kept as the lowest version of golang that we support - pull: always - environment: - GOPROXY: https://goproxy.io - commands: - - go build -o gitea_no_gcc # test if build succeeds without the sqlite tag - depends_on: [deps-backend, checks-backend] - volumes: - - name: deps - path: /go - - - name: build-backend-arm64 - image: gitea/test_env:linux-1.20-amd64 - environment: - GOPROXY: https://goproxy.io - GOOS: linux - GOARCH: arm64 - TAGS: bindata gogit - commands: - - make backend # test cross compile - - rm ./gitea # clean - depends_on: [deps-backend, checks-backend] - volumes: - - name: deps - path: /go - - - name: build-backend-windows - image: gitea/test_env:linux-1.20-amd64 - environment: - GOPROXY: https://goproxy.io - GOOS: windows - GOARCH: amd64 - TAGS: bindata gogit - commands: - - go build -o gitea_windows - depends_on: [deps-backend, checks-backend] - volumes: - - name: deps - path: /go - - - name: build-backend-386 - image: gitea/test_env:linux-1.20-amd64 - environment: - GOPROXY: https://goproxy.io - GOOS: linux - GOARCH: 386 - commands: - - go build -o gitea_linux_386 # test if compatible with 32 bit - depends_on: [deps-backend, checks-backend] - volumes: - - name: deps - path: /go - ---- -kind: pipeline -type: docker -name: testing-pgsql - -platform: - os: linux - arch: amd64 - -depends_on: - - compliance - -trigger: - event: - - pull_request - paths: - exclude: - - "docs/**" - -volumes: - - name: deps - temp: {} - -services: - - name: pgsql - pull: default - image: postgres:15 - environment: - POSTGRES_DB: test - POSTGRES_PASSWORD: postgres - - - name: ldap - image: gitea/test-openldap:latest - pull: always - - - name: minio - image: minio/minio:RELEASE.2021-03-12T00-00-47Z - pull: always - commands: - - minio server /data - environment: - MINIO_ACCESS_KEY: 123456 - MINIO_SECRET_KEY: 12345678 - -steps: - - name: fetch-tags - image: docker:git - pull: always - commands: - - git fetch --tags --force - when: - event: - exclude: - - pull_request - - - name: deps-backend - image: gitea/test_env:linux-1.20-amd64 - pull: always - commands: - - make deps-backend - volumes: - - name: deps - path: /go - - - name: prepare-test-env - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - pull: always - commands: - - ./build/test-env-prepare.sh - - - name: build - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - ./build/test-env-check.sh - - make backend - environment: - GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not - GOSUMDB: sum.golang.org - TAGS: bindata - depends_on: [deps-backend, prepare-test-env] - volumes: - - name: deps - path: /go - - - name: test-pgsql - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - timeout -s ABRT 50m make test-pgsql-migration test-pgsql - environment: - GOPROXY: https://goproxy.io - TAGS: bindata gogit - RACE_ENABLED: true - TEST_TAGS: gogit - TEST_LDAP: 1 - USE_REPO_TEST_DIR: 1 - depends_on: [build] - volumes: - - name: deps - path: /go - ---- -kind: pipeline -type: docker -name: testing-mysql - -platform: - os: linux - arch: amd64 - -depends_on: - - compliance - -trigger: - event: - - pull_request - paths: - exclude: - - "docs/**" - -volumes: - - name: deps - temp: {} - -services: - - name: mysql - image: mysql:5.7 - pull: always - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: test - - - name: elasticsearch - image: elasticsearch:7.5.0 - pull: always - environment: - discovery.type: single-node - - - name: smtpimap - image: tabascoterrier/docker-imap-devel:latest - pull: always - -steps: - - name: fetch-tags - image: docker:git - pull: always - commands: - - git fetch --tags --force - when: - event: - exclude: - - pull_request - - - name: deps-backend - image: gitea/test_env:linux-1.20-amd64 - pull: always - commands: - - make deps-backend - volumes: - - name: deps - path: /go - - - name: prepare-test-env - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - pull: always - commands: - - ./build/test-env-prepare.sh - - - name: build - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - ./build/test-env-check.sh - - make backend - environment: - GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not - GOSUMDB: sum.golang.org - TAGS: bindata - depends_on: [deps-backend, prepare-test-env] - volumes: - - name: deps - path: /go - - - name: unit-test - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - make unit-test-coverage test-check - environment: - GOPROXY: https://goproxy.io - TAGS: bindata - RACE_ENABLED: true - GITHUB_READ_TOKEN: - from_secret: github_read_token - depends_on: [deps-backend, prepare-test-env] - volumes: - - name: deps - path: /go - - - name: unit-test-gogit - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - make unit-test-coverage test-check - environment: - GOPROXY: https://goproxy.io - TAGS: bindata gogit - RACE_ENABLED: true - GITHUB_READ_TOKEN: - from_secret: github_read_token - depends_on: [deps-backend, prepare-test-env] - volumes: - - name: deps - path: /go - - - name: test-mysql - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - make test-mysql-migration integration-test-coverage - environment: - GOPROXY: https://goproxy.io - TAGS: bindata - RACE_ENABLED: true - USE_REPO_TEST_DIR: 1 - TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" - depends_on: [build] - volumes: - - name: deps - path: /go - - - name: generate-coverage - image: gitea/test_env:linux-1.20-amd64 - commands: - - make coverage - environment: - GOPROXY: https://goproxy.io - TAGS: bindata - depends_on: [unit-test, test-mysql] - when: - branch: - - main - event: - - push - - pull_request - - - name: coverage-codecov - image: woodpeckerci/plugin-codecov:next-alpine - pull: always - settings: - files: - - coverage.all - token: - from_secret: codecov_token - depends_on: [generate-coverage] - when: - branch: - - main - event: - - push - - pull_request - ---- -kind: pipeline -type: docker -name: testing-mysql8 - -platform: - os: linux - arch: amd64 - -depends_on: - - compliance - -trigger: - event: - - pull_request - paths: - exclude: - - "docs/**" - -volumes: - - name: deps - temp: {} - -services: - - name: mysql8 - image: mysql:8 - pull: always - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: testgitea - -steps: - - name: fetch-tags - image: docker:git - pull: always - commands: - - git fetch --tags --force - when: - event: - exclude: - - pull_request - - - name: deps-backend - image: gitea/test_env:linux-1.20-amd64 - pull: always - commands: - - make deps-backend - volumes: - - name: deps - path: /go - - - name: prepare-test-env - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - pull: always - commands: - - ./build/test-env-prepare.sh - - - name: build - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - ./build/test-env-check.sh - - make backend - environment: - GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not - GOSUMDB: sum.golang.org - TAGS: bindata - depends_on: [deps-backend, prepare-test-env] - volumes: - - name: deps - path: /go - - - name: test-mysql8 - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - timeout -s ABRT 50m make test-mysql8-migration test-mysql8 - environment: - GOPROXY: https://goproxy.io - TAGS: bindata - USE_REPO_TEST_DIR: 1 - depends_on: [build] - volumes: - - name: deps - path: /go - ---- -kind: pipeline -type: docker -name: testing-mssql - -platform: - os: linux - arch: amd64 - -depends_on: - - compliance - -trigger: - event: - - pull_request - paths: - exclude: - - "docs/**" - -volumes: - - name: deps - temp: {} - -services: - - name: mssql - image: mcr.microsoft.com/mssql/server:latest - pull: always - environment: - ACCEPT_EULA: Y - MSSQL_PID: Standard - SA_PASSWORD: MwantsaSecurePassword1 - -steps: - - name: fetch-tags - image: docker:git - pull: always - commands: - - git fetch --tags --force - when: - event: - exclude: - - pull_request - - - name: deps-backend - image: gitea/test_env:linux-1.20-amd64 - pull: always - commands: - - make deps-backend - volumes: - - name: deps - path: /go - - - name: prepare-test-env - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - pull: always - commands: - - ./build/test-env-prepare.sh - - - name: build - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - ./build/test-env-check.sh - - make backend - environment: - GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not - GOSUMDB: sum.golang.org - TAGS: bindata - depends_on: [deps-backend, prepare-test-env] - volumes: - - name: deps - path: /go - - - name: test-mssql - image: gitea/test_env:linux-1.20-amd64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - make test-mssql-migration test-mssql - environment: - GOPROXY: https://goproxy.io - TAGS: bindata - USE_REPO_TEST_DIR: 1 - depends_on: [build] - volumes: - - name: deps - path: /go - ---- -kind: pipeline -name: testing-sqlite - -platform: - os: linux - arch: arm64 - -depends_on: - - compliance - -trigger: - event: - - pull_request - paths: - exclude: - - "docs/**" - -volumes: - - name: deps - temp: {} - -steps: - - name: fetch-tags - image: docker:git - pull: always - commands: - - git fetch --tags --force - when: - event: - exclude: - - pull_request - - - name: deps-backend - image: gitea/test_env:linux-1.20-arm64 - pull: always - commands: - - make deps-backend - volumes: - - name: deps - path: /go - - - name: prepare-test-env - image: gitea/test_env:linux-1.20-arm64 # https://gitea.com/gitea/test-env - pull: always - commands: - - ./build/test-env-prepare.sh - - - name: build - image: gitea/test_env:linux-1.20-arm64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - ./build/test-env-check.sh - - make backend - environment: - GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not - GOSUMDB: sum.golang.org - TAGS: bindata gogit sqlite sqlite_unlock_notify - depends_on: [deps-backend, prepare-test-env] - volumes: - - name: deps - path: /go - - - name: test-sqlite - image: gitea/test_env:linux-1.20-arm64 # https://gitea.com/gitea/test-env - user: gitea - commands: - - timeout -s ABRT 50m make test-sqlite-migration test-sqlite - environment: - GOPROXY: https://goproxy.io - TAGS: bindata gogit sqlite sqlite_unlock_notify - RACE_ENABLED: true - TEST_TAGS: gogit sqlite sqlite_unlock_notify - USE_REPO_TEST_DIR: 1 - depends_on: [build] - volumes: - - name: deps - path: /go - ---- -kind: pipeline -type: docker -name: testing-e2e - -platform: - os: linux - arch: amd64 - -depends_on: - - compliance - -trigger: - event: - - pull_request - paths: - exclude: - - "docs/**" - -volumes: - - name: deps - temp: {} - -steps: - - name: deps-frontend - image: node:18 - pull: always - commands: - - make deps-frontend - - - name: build-frontend - image: node:18 - commands: - - make frontend - depends_on: [deps-frontend] - - - name: deps-backend - image: gitea/test_env:linux-1.20-amd64 - pull: always - commands: - - make deps-backend - volumes: - - name: deps - path: /go - - # TODO: We should probably build all dependencies into a test image - - name: test-e2e - image: mcr.microsoft.com/playwright:v1.32.3-focal - commands: - - apt-get -qq update && apt-get -qqy install jq build-essential - - curl -fsSL "https://go.dev/dl/$(curl -s 'https://go.dev/dl/?mode=json' | jq -r '.[].version' | sort -Vr | head -1).linux-amd64.tar.gz" | tar -xz -C /usr/local - - groupadd --gid 1001 gitea && useradd -m --gid 1001 --uid 1001 gitea - - ./build/test-env-prepare.sh - - su gitea bash -c "export PATH=$PATH:/usr/local/go/bin && timeout -s ABRT 40m make test-e2e-sqlite" - environment: - GOPROXY: https://goproxy.io - GOSUMDB: sum.golang.org - USE_REPO_TEST_DIR: 1 - DEBIAN_FRONTEND: noninteractive - depends_on: [build-frontend, deps-backend] - volumes: - - name: deps - path: /go - --- kind: pipeline type: docker @@ -767,13 +21,6 @@ trigger: exclude: - "docs/**" -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - volumes: - name: deps temp: {} @@ -786,7 +33,7 @@ steps: - git fetch --tags --force - name: deps-frontend - image: node:18 + image: node:20 pull: always commands: - make deps-frontend @@ -804,7 +51,7 @@ steps: image: techknowlogick/xgo:go-1.20.x pull: always commands: - # Upgrade to node 18 once https://github.com/techknowlogick/xgo/issues/163 is resolved + # Upgrade to node 20 once https://github.com/techknowlogick/xgo/issues/163 is resolved - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs - export PATH=$PATH:$GOPATH/bin - make release @@ -902,13 +149,6 @@ trigger: event: - tag -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - volumes: - name: deps temp: {} @@ -921,7 +161,7 @@ steps: - git fetch --tags --force - name: deps-frontend - image: node:18 + image: node:20 pull: always commands: - make deps-frontend @@ -939,7 +179,7 @@ steps: image: techknowlogick/xgo:go-1.20.x pull: always commands: - # Upgrade to node 18 once https://github.com/techknowlogick/xgo/issues/163 is resolved + # Upgrade to node 20 once https://github.com/techknowlogick/xgo/issues/163 is resolved - curl -sL https://deb.nodesource.com/setup_16.x | bash - && apt-get -qqy install nodejs - export PATH=$PATH:$GOPATH/bin - make release @@ -1013,22 +253,12 @@ platform: os: linux arch: amd64 -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - trigger: ref: include: - "refs/tags/**" exclude: - "refs/tags/**-rc*" - event: - exclude: - - cron paths: exclude: - "docs/**" @@ -1093,19 +323,9 @@ platform: os: linux arch: amd64 -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - trigger: ref: - "refs/tags/**-rc*" - event: - exclude: - - cron paths: exclude: - "docs/**" @@ -1168,19 +388,9 @@ platform: os: linux arch: amd64 -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - trigger: ref: - refs/heads/main - event: - exclude: - - cron steps: - name: fetch-tags @@ -1241,19 +451,9 @@ platform: os: linux arch: amd64 -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - trigger: ref: - "refs/heads/release/v*" - event: - exclude: - - cron steps: - name: fetch-tags @@ -1315,22 +515,12 @@ platform: os: linux arch: arm64 -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - trigger: ref: include: - "refs/tags/**" exclude: - "refs/tags/**-rc*" - event: - exclude: - - cron paths: exclude: - "docs/**" @@ -1395,19 +585,9 @@ platform: os: linux arch: arm64 -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - trigger: ref: - "refs/tags/**-rc*" - event: - exclude: - - cron paths: exclude: - "docs/**" @@ -1470,19 +650,9 @@ platform: os: linux arch: arm64 -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - trigger: ref: - refs/heads/main - event: - exclude: - - cron paths: exclude: - "docs/**" @@ -1546,19 +716,9 @@ platform: os: linux arch: arm64 -depends_on: - - testing-mysql - - testing-mysql8 - - testing-mssql - - testing-pgsql - - testing-sqlite - trigger: ref: - "refs/heads/release/v*" - event: - exclude: - - cron steps: - name: fetch-tags @@ -1647,9 +807,6 @@ steps: trigger: ref: - "refs/tags/**" - event: - exclude: - - cron paths: exclude: - "docs/**" @@ -1697,9 +854,6 @@ trigger: ref: - refs/heads/main - "refs/heads/release/v*" - event: - exclude: - - cron paths: exclude: - "docs/**" diff --git a/.eslintrc.yaml b/.eslintrc.yaml index 878b74cba4d..c4e0b9de8cb 100644 --- a/.eslintrc.yaml +++ b/.eslintrc.yaml @@ -113,9 +113,9 @@ rules: import/namespace: [0] import/newline-after-import: [0] import/no-absolute-path: [0] - import/no-amd: [0] + import/no-amd: [2] import/no-anonymous-default-export: [0] - import/no-commonjs: [0] + import/no-commonjs: [2] import/no-cycle: [2, {ignoreExternal: true, maxDepth: 1}] import/no-default-export: [0] import/no-deprecated: [0] @@ -576,7 +576,7 @@ rules: sonarjs/no-nested-template-literals: [0] sonarjs/no-one-iteration-loop: [2] sonarjs/no-redundant-boolean: [2] - sonarjs/no-redundant-jump: [0] + sonarjs/no-redundant-jump: [2] sonarjs/no-same-line-conditional: [2] sonarjs/no-small-switch: [0] sonarjs/no-unused-collection: [2] @@ -616,17 +616,18 @@ rules: unicorn/import-style: [0] unicorn/new-for-builtins: [2] unicorn/no-abusive-eslint-disable: [0] + unicorn/no-array-callback-reference: [0] unicorn/no-array-for-each: [2] - unicorn/no-array-instanceof: [0] unicorn/no-array-method-this-argument: [2] unicorn/no-array-push-push: [2] + unicorn/no-array-reduce: [2] unicorn/no-await-expression-member: [0] unicorn/no-console-spaces: [0] unicorn/no-document-cookie: [2] unicorn/no-empty-file: [2] - unicorn/no-fn-reference-in-iterator: [0] unicorn/no-for-loop: [0] unicorn/no-hex-escape: [0] + unicorn/no-instanceof-array: [0] unicorn/no-invalid-remove-event-listener: [2] unicorn/no-keyword-prefix: [0] unicorn/no-lonely-if: [2] @@ -637,7 +638,6 @@ rules: unicorn/no-null: [0] unicorn/no-object-as-default-parameter: [0] unicorn/no-process-exit: [0] - unicorn/no-reduce: [2] unicorn/no-static-only-class: [2] unicorn/no-thenable: [2] unicorn/no-this-assignment: [2] @@ -663,15 +663,19 @@ rules: unicorn/prefer-array-index-of: [2] unicorn/prefer-array-some: [2] unicorn/prefer-at: [0] + unicorn/prefer-blob-reading-methods: [2] unicorn/prefer-code-point: [0] - unicorn/prefer-dataset: [2] unicorn/prefer-date-now: [2] unicorn/prefer-default-parameters: [0] - unicorn/prefer-event-key: [2] + unicorn/prefer-dom-node-append: [2] + unicorn/prefer-dom-node-dataset: [0] + unicorn/prefer-dom-node-remove: [2] + unicorn/prefer-dom-node-text-content: [2] unicorn/prefer-event-target: [2] unicorn/prefer-export-from: [2, {ignoreUsedVariables: true}] unicorn/prefer-includes: [2] unicorn/prefer-json-parse-buffer: [0] + unicorn/prefer-keyboard-event-key: [2] unicorn/prefer-logical-operator-over-ternary: [2] unicorn/prefer-math-trunc: [2] unicorn/prefer-modern-dom-apis: [0] @@ -679,9 +683,7 @@ rules: unicorn/prefer-module: [2] unicorn/prefer-native-coercion-functions: [2] unicorn/prefer-negative-index: [2] - unicorn/prefer-node-append: [0] unicorn/prefer-node-protocol: [2] - unicorn/prefer-node-remove: [0] unicorn/prefer-number-properties: [0] unicorn/prefer-object-from-entries: [2] unicorn/prefer-object-has-own: [0] @@ -690,17 +692,17 @@ rules: unicorn/prefer-query-selector: [0] unicorn/prefer-reflect-apply: [0] unicorn/prefer-regexp-test: [2] - unicorn/prefer-replace-all: [0] unicorn/prefer-set-has: [0] unicorn/prefer-set-size: [2] unicorn/prefer-spread: [0] - unicorn/prefer-starts-ends-with: [2] + unicorn/prefer-string-replace-all: [0] unicorn/prefer-string-slice: [0] + unicorn/prefer-string-starts-ends-with: [2] + unicorn/prefer-string-trim-start-end: [2] unicorn/prefer-switch: [0] unicorn/prefer-ternary: [0] unicorn/prefer-text-content: [2] unicorn/prefer-top-level-await: [0] - unicorn/prefer-trim-start-end: [2] unicorn/prefer-type-error: [0] unicorn/prevent-abbreviations: [0] unicorn/relative-url-style: [2] diff --git a/.github/lock.yml b/.github/lock.yml deleted file mode 100644 index 6beadcaf110..00000000000 --- a/.github/lock.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Configuration for Lock Threads - https://github.com/dessant/lock-threads-app - -# Number of days of inactivity before a closed issue or pull request is locked -daysUntilLock: 60 - -# Skip issues and pull requests created before a given timestamp. Timestamp must -# follow ISO 8601 (`YYYY-MM-DD`). `false` is disabled -skipCreatedBefore: false - -# Issues and pull requests with these labels will be ignored. -exemptLabels: [] - -# Label to add before locking, such as `outdated`. `false` is disabled -lockLabel: false - -# Comment to post before locking. -lockComment: > - This thread has been automatically locked since there has not been - any recent activity after it was closed. Please open a new issue for - related bugs and link to relevant comments in this thread. - -# Assign `resolved` as the reason for locking. Set to `false` to disable -setLockReason: true diff --git a/.github/workflows/cron-licenses.yml b/.github/workflows/cron-licenses.yml index 116b8dd0956..72390787104 100644 --- a/.github/workflows/cron-licenses.yml +++ b/.github/workflows/cron-licenses.yml @@ -1,10 +1,10 @@ +name: "Cron: Update licenses and gitignores" + on: schedule: # weekly on Monday at 0:07 UTC - cron: "7 0 * * 1" -name: Update licenses and gitignores - jobs: cron: runs-on: ubuntu-latest diff --git a/.github/workflows/cron-translations.yml b/.github/workflows/cron-translations.yml index 31262fd3dde..c83450d2e66 100644 --- a/.github/workflows/cron-translations.yml +++ b/.github/workflows/cron-translations.yml @@ -1,9 +1,9 @@ +name: "Cron: Pull translations from Crowdin" + on: schedule: - cron: "7 0 * * *" # every day at 0:07 UTC -name: Pull translations from Crowdin - jobs: crowdin_pull: runs-on: ubuntu-latest diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 00000000000..2e132b95fed --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,21 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '0 0 * * *' # Run once a day + workflow_dispatch: + +permissions: + issues: write + pull-requests: write + +concurrency: + group: lock + +jobs: + action: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v4 + with: + issue-inactive-days: 45 diff --git a/.github/workflows/pull-compliance.yml b/.github/workflows/pull-compliance.yml new file mode 100644 index 00000000000..155d9298e96 --- /dev/null +++ b/.github/workflows/pull-compliance.yml @@ -0,0 +1,142 @@ +name: "Pull: Compliance Tests" + +on: [pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + lint_basic: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + check-latest: true + - name: deps-backend + run: make deps-backend deps-tools + - name: lint backend + run: make lint-backend + env: + GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata sqlite sqlite_unlock_notify + lint_windows: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + check-latest: true + - name: deps-backend + run: make deps-backend deps-tools + - name: lint-backend-windows + run: make lint-go-windows lint-go-vet + env: + GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata sqlite sqlite_unlock_notify + GOOS: windows + GOARCH: amd64 + lint_gogit: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + check-latest: true + - name: deps-backend + run: make deps-backend deps-tools + - name: lint-backend-gogit + run: make lint-backend + env: + GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata gogit sqlite sqlite_unlock_notify + - name: checks backend + run: make --always-make checks-backend # ensure the 'go-licenses' make target runs + check_backend: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + check-latest: true + - name: deps-backend + run: make deps-backend deps-tools + - name: checks backend + run: make --always-make checks-backend # ensure the 'go-licenses' make target runs + frontend: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup node + uses: actions/setup-node@v3 + with: + node-version: 20 + - name: deps-frontend + run: make deps-frontend + - name: lint frontend + run: make lint-frontend + - name: checks frontend + run: make checks-frontend + - name: test frontend + run: make test-frontend + backend: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + check-latest: true + - name: setup node + uses: actions/setup-node@v3 + with: + node-version: 20 + - name: deps-backend + run: make deps-backend deps-tools + - name: deps-frontend + run: make deps-frontend + - name: build frontend + run: make frontend + - name: build-backend-no-gcc + run: go build -o gitea_no_gcc # test if build succeeds without the sqlite tag + env: + GOPROXY: https://goproxy.io + - name: build-backend-arm64 + run: make backend # test cross compile + env: + GOPROXY: https://goproxy.io + GOOS: linux + GOARCH: arm64 + TAGS: bindata gogit + - name: build-backend-windows + run: go build -o gitea_windows + env: + GOPROXY: https://goproxy.io + GOOS: windows + GOARCH: amd64 + TAGS: bindata gogit + - name: build-backend-386 + run: go build -o gitea_linux_386 # test if compatible with 32 bit + env: + GOPROXY: https://goproxy.io + GOOS: linux + GOARCH: 386 diff --git a/.github/workflows/pull-compliance_docs.yml b/.github/workflows/pull-compliance_docs.yml index e3c3a425412..c033b62711d 100644 --- a/.github/workflows/pull-compliance_docs.yml +++ b/.github/workflows/pull-compliance_docs.yml @@ -1,4 +1,4 @@ -name: Compliance testing for documentation +name: "Pull: Compliance testing for documentation" on: pull_request: @@ -6,6 +6,10 @@ on: - "docs/**" - "*.md" +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: compliance-docs: runs-on: ubuntu-latest @@ -13,9 +17,9 @@ jobs: - name: checkout uses: actions/checkout@v3 - name: setup node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: - node-version: 18 + node-version: 20 - name: install dependencies run: make deps-frontend - name: lint markdown diff --git a/.github/workflows/pull-db_test.yml b/.github/workflows/pull-db_test.yml new file mode 100644 index 00000000000..243499b611f --- /dev/null +++ b/.github/workflows/pull-db_test.yml @@ -0,0 +1,274 @@ +name: "Pull: Database Tests" + +on: [pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + # PostgreSQL Tests + db_pgsql_test: + runs-on: ubuntu-latest + services: + pgsql: + image: postgres:15 + env: + POSTGRES_DB: test + POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" + ldap: + image: gitea/test-openldap:latest + ports: + - "389:389" + - "636:636" + minio: + # as github actions doesn't support "entrypoint", we need to use a non-official image + # that has a custom entrypoint set to "minio server /data" + image: bitnami/minio:2021.3.17 + env: + MINIO_ACCESS_KEY: 123456 + MINIO_SECRET_KEY: 12345678 + ports: + - "9000:9000" + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20.0' + - name: Add hosts to /etc/hosts + run: echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts + - name: install dependencies + run: make deps-backend + - name: build + run: make backend + env: + GOPROXY: https://goproxy.io + GOSUMDB: sum.golang.org + TAGS: bindata + - name: run tests + run: timeout -s ABRT 50m make test-pgsql-migration test-pgsql + env: + GOPROXY: https://goproxy.io + TAGS: bindata gogit + RACE_ENABLED: true + TEST_TAGS: gogit + TEST_LDAP: 1 + USE_REPO_TEST_DIR: 1 + + # SQLite Tests + db_sqlite_test: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20.0' + - name: install dependencies + run: make deps-backend + - name: build + run: make backend + env: + GOPROXY: https://goproxy.io + GOSUMDB: sum.golang.org + TAGS: bindata gogit sqlite sqlite_unlock_notify + - name: run tests + run: timeout -s ABRT 50m make test-sqlite-migration test-sqlite + env: + GOPROXY: https://goproxy.io + TAGS: bindata gogit sqlite sqlite_unlock_notify + RACE_ENABLED: true + TEST_TAGS: gogit sqlite sqlite_unlock_notify + USE_REPO_TEST_DIR: 1 + + # Unit Tests + db_unit_tests: + runs-on: ubuntu-latest + services: + mysql: + image: mysql:5.7 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: test + ports: + - "3306:3306" + elasticsearch: + image: elasticsearch:7.5.0 + env: + discovery.type: single-node + ports: + - "9200:9200" + smtpimap: + image: tabascoterrier/docker-imap-devel:latest + ports: + - "25:25" + - "143:143" + - "587:587" + - "993:993" + redis: + image: redis + # Set health checks to wait until redis has started + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 3s + --health-retries 10 + ports: + - 6379:6379 + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20.0' + - name: Add hosts to /etc/hosts + run: echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts + - name: install dependencies + run: make deps-backend + - name: build + run: make backend + env: + GOPROXY: https://goproxy.io + GOSUMDB: sum.golang.org + TAGS: bindata + - name: unit tests + run: make unit-test-coverage test-check + env: + GOPROXY: https://goproxy.io + TAGS: bindata + RACE_ENABLED: true + GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} + - name: unit tests (gogit) + run: make unit-test-coverage test-check + env: + GOPROXY: https://goproxy.io + TAGS: bindata gogit + RACE_ENABLED: true + GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} + + # MySQL Tests + db_mysql_test: + runs-on: ubuntu-latest + services: + mysql: + image: mysql:5.7 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: test + ports: + - "3306:3306" + elasticsearch: + image: elasticsearch:7.5.0 + env: + discovery.type: single-node + ports: + - "9200:9200" + smtpimap: + image: tabascoterrier/docker-imap-devel:latest + ports: + - "25:25" + - "143:143" + - "587:587" + - "993:993" + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20.0' + - name: Add hosts to /etc/hosts + run: echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts + - name: install dependencies + run: make deps-backend + - name: build + run: make backend + env: + GOPROXY: https://goproxy.io + GOSUMDB: sum.golang.org + TAGS: bindata + - name: run tests + run: make test-mysql-migration integration-test-coverage + env: + GOPROXY: https://goproxy.io + TAGS: bindata + RACE_ENABLED: true + USE_REPO_TEST_DIR: 1 + TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" + + # MySQL8 Tests + db_mysql8_test: + runs-on: ubuntu-latest + services: + mysql8: + image: mysql:8 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: testgitea + ports: + - "3306:3306" + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20.0' + - name: Add hosts to /etc/hosts + run: echo "127.0.0.1 mysql8" | sudo tee -a /etc/hosts + - name: install dependencies + run: make deps-backend + - name: build + run: make backend + env: + GOPROXY: https://goproxy.io + GOSUMDB: sum.golang.org + TAGS: bindata + - name: run tests + run: timeout -s ABRT 50m make test-mysql8-migration test-mysql8 + env: + GOPROXY: https://goproxy.io + TAGS: bindata + USE_REPO_TEST_DIR: 1 + + # MSSQL Tests + db_mssql_test: + runs-on: ubuntu-latest + services: + mssql: + image: mcr.microsoft.com/mssql/server:latest + env: + ACCEPT_EULA: Y + MSSQL_PID: Standard + SA_PASSWORD: MwantsaSecurePassword1 + ports: + - "1433:1433" + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20.0' + - name: Add hosts to /etc/hosts + run: echo "127.0.0.1 mssql" | sudo tee -a /etc/hosts + - name: install dependencies + run: make deps-backend + - name: build + run: make backend + env: + GOPROXY: https://goproxy.io + GOSUMDB: sum.golang.org + TAGS: bindata + - name: run tests + run: timeout -s ABRT 50m make test-mssql-migration test-mssql + env: + GOPROXY: https://goproxy.io + TAGS: bindata + USE_REPO_TEST_DIR: 1 diff --git a/.github/workflows/pull-docker_dryrun.yml b/.github/workflows/pull-docker_dryrun.yml index 719668db819..f17d6014b60 100644 --- a/.github/workflows/pull-docker_dryrun.yml +++ b/.github/workflows/pull-docker_dryrun.yml @@ -1,7 +1,11 @@ -name: Docker build dry run +name: "Pull: Docker Dry Run" on: [pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + jobs: docker_dryrun: runs-on: ubuntu-latest diff --git a/.github/workflows/pull-e2e.yml b/.github/workflows/pull-e2e.yml new file mode 100644 index 00000000000..79c54175080 --- /dev/null +++ b/.github/workflows/pull-e2e.yml @@ -0,0 +1,33 @@ +name: "Pull: E2E Tests" + +on: [pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + e2e_tests: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: setup go + uses: actions/setup-go@v4 + with: + go-version: '>=1.20' + check-latest: true + - name: setup node + uses: actions/setup-node@v3 + with: + node-version: 20 + - name: build + run: make deps-frontend frontend deps-backend + - name: Install playwright browsers + run: npx playwright install --with-deps + - name: run tests + run: timeout -s ABRT 40m make test-e2e-sqlite + env: + GOPROXY: https://goproxy.io + GOSUMDB: sum.golang.org + USE_REPO_TEST_DIR: 1 diff --git a/.github/workflows/push-publish_docs.yml b/.github/workflows/push-publish_docs.yml index 579157ccd8b..9037e278ca1 100644 --- a/.github/workflows/push-publish_docs.yml +++ b/.github/workflows/push-publish_docs.yml @@ -1,4 +1,4 @@ -name: Publish documentation +name: "Docs: Publish" on: push: diff --git a/.gitpod.yml b/.gitpod.yml index 442f434eaaa..f1b2fb19571 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -31,7 +31,7 @@ vscode: - golang.go - stylelint.vscode-stylelint - DavidAnson.vscode-markdownlint - - johnsoncodehk.volar + - Vue.volar - ms-azuretools.vscode-docker - zixuanchen.vitest-explorer - alexcvzz.vscode-sqlite diff --git a/CHANGELOG.md b/CHANGELOG.md index c9d1bab896c..e4e5c9368a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ This changelog goes through all the changes that have been made in each release without substantial changes to our git log; to see the highlights of what has been added to each release, please refer to the [blog](https://blog.gitea.io). +## [1.19.3](https://github.com/go-gitea/gitea/releases/tag/1.19.3) - 2023-05-03 + +* SECURITY + * Use golang 1.20.4 to fix CVE-2023-24539, CVE-2023-24540, and CVE-2023-29400 +* ENHANCEMENTS + * Enable whitespace rendering on selection in Monaco (#24444) (#24485) + * Improve milestone filter on issues page (#22423) (#24440) +* BUGFIXES + * Fix api error message if fork exists (#24487) (#24493) + * Fix user-cards format (#24428) (#24431) + * Fix incorrect CurrentUser check for docker rootless (#24435) + * Getting the tag list does not require being signed in (#24413) (#24416) + ## [1.19.2](https://github.com/go-gitea/gitea/releases/tag/1.19.2) - 2023-04-26 * SECURITY diff --git a/Makefile b/Makefile index 5192d2c5a6b..6a03875b793 100644 --- a/Makefile +++ b/Makefile @@ -139,7 +139,7 @@ GO_DIRS := build cmd models modules routers services tests WEB_DIRS := web_src/js web_src/css GO_SOURCES := $(wildcard *.go) -GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" -not -path modules/options/bindata.go -not -path modules/public/bindata.go -not -path modules/templates/bindata.go) +GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" ! -path modules/options/bindata.go ! -path modules/public/bindata.go ! -path modules/templates/bindata.go) GO_SOURCES += $(GENERATED_GO_DEST) GO_SOURCES_NO_BINDATA := $(GO_SOURCES) diff --git a/README.md b/README.md index 29d5371a368..0ee17722862 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Translations are done through Crowdin. If you want to translate to a new languag You can also just create an issue for adding a language or ask on discord on the #translation channel. If you need context or find some translation issues, you can leave a comment on the string or ask on Discord. For general translation questions there is a section in the docs. Currently a bit empty but we hope to fill it as questions pop up. -https://docs.gitea.io/en-us/translation-guidelines/ +https://docs.gitea.io/en-us/contributing/translation-guidelines/ [![Crowdin](https://badges.crowdin.net/gitea/localized.svg)](https://crowdin.com/project/gitea) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 57285b6a922..49f18e7de70 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -69,6 +69,21 @@ "path": "github.com/Azure/go-ntlmssp/LICENSE", "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2016 Microsoft\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, + { + "name": "github.com/ClickHouse/ch-go", + "path": "github.com/ClickHouse/ch-go/LICENSE", + "licenseText": "Copyright 2016-2023 ClickHouse, Inc.\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2016-2023 ClickHouse, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, + { + "name": "github.com/ClickHouse/clickhouse-go/v2", + "path": "github.com/ClickHouse/clickhouse-go/v2/LICENSE", + "licenseText": "Copyright 2016-2023 ClickHouse, Inc.\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2016-2023 ClickHouse, Inc.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, + { + "name": "github.com/DataDog/zstd", + "path": "github.com/DataDog/zstd/LICENSE", + "licenseText": "Simplified BSD License\n\nCopyright (c) 2016, Datadog \u003cinfo@datadoghq.com\u003e\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n * Redistributions of source code must retain the above copyright notice,\n this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n * Neither the name of the copyright holder nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, { "name": "github.com/NYTimes/gziphandler", "path": "github.com/NYTimes/gziphandler/LICENSE", @@ -399,6 +414,16 @@ "path": "github.com/go-enry/go-enry/v2/LICENSE", "licenseText": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License." }, + { + "name": "github.com/go-faster/city", + "path": "github.com/go-faster/city/LICENSE", + "licenseText": "MIT License\n\nCopyright (c) 2018 tenfy\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, + { + "name": "github.com/go-faster/errors", + "path": "github.com/go-faster/errors/LICENSE", + "licenseText": "Copyright (c) 2009 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" + }, { "name": "github.com/go-fed/httpsig", "path": "github.com/go-fed/httpsig/LICENSE", @@ -657,7 +682,7 @@ { "name": "github.com/klauspost/pgzip", "path": "github.com/klauspost/pgzip/LICENSE", - "licenseText": "MIT License\n\nCopyright (c) 2014 Klaus Post\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2014 Klaus Post\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\n" }, { "name": "github.com/lib/pq", @@ -727,7 +752,7 @@ { "name": "github.com/miekg/dns", "path": "github.com/miekg/dns/LICENSE", - "licenseText": "Copyright (c) 2009 The Go Authors. All rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are\nmet:\n\n * Redistributions of source code must retain the above copyright\nnotice, this list of conditions and the following disclaimer.\n * Redistributions in binary form must reproduce the above\ncopyright notice, this list of conditions and the following disclaimer\nin the documentation and/or other materials provided with the\ndistribution.\n * Neither the name of Google Inc. nor the names of its\ncontributors may be used to endorse or promote products derived from\nthis software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n\"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\nLIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\nA PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\nOWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\nSPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\nLIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\nDATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\nTHEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nAs this is fork of the official Go code the same license applies.\nExtensions of the original work are copyright (c) 2011 Miek Gieben\n" + "licenseText": "BSD 3-Clause License\n\nCopyright (c) 2009, The Go Authors. Extensions copyright (c) 2011, Miek Gieben. \nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n3. Neither the name of the copyright holder nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" }, { "name": "github.com/minio/md5-simd", @@ -809,11 +834,21 @@ "path": "github.com/opencontainers/image-spec/specs-go/LICENSE", "licenseText": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n Copyright 2016 The Linux Foundation.\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, + { + "name": "github.com/paulmach/orb", + "path": "github.com/paulmach/orb/LICENSE.md", + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2017 Paul Mach\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" + }, { "name": "github.com/pierrec/lz4/v4", "path": "github.com/pierrec/lz4/v4/LICENSE", "licenseText": "Copyright (c) 2015, Pierre Curto\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met:\n\n* Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer.\n\n* Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution.\n\n* Neither the name of xxHash nor the names of its\n contributors may be used to endorse or promote products derived from\n this software without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR\nSERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER\nCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,\nOR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\nOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\n" }, + { + "name": "github.com/pjbgf/sha1cd", + "path": "github.com/pjbgf/sha1cd/LICENSE", + "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, { "name": "github.com/pkg/errors", "path": "github.com/pkg/errors/LICENSE", @@ -884,11 +919,26 @@ "path": "github.com/santhosh-tekuri/jsonschema/v5/LICENSE", "licenseText": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability." }, + { + "name": "github.com/sassoftware/go-rpmutils", + "path": "github.com/sassoftware/go-rpmutils/LICENSE", + "licenseText": "\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, + { + "name": "github.com/segmentio/asm", + "path": "github.com/segmentio/asm/LICENSE", + "licenseText": "MIT License\n\nCopyright (c) 2021 Segment\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, { "name": "github.com/sergi/go-diff/diffmatchpatch", "path": "github.com/sergi/go-diff/diffmatchpatch/LICENSE", "licenseText": "Copyright (c) 2012-2016 The go-diff Authors. All rights reserved.\n\nPermission is hereby granted, free of charge, to any person obtaining a\ncopy of this software and associated documentation files (the \"Software\"),\nto deal in the Software without restriction, including without limitation\nthe rights to use, copy, modify, merge, publish, distribute, sublicense,\nand/or sell copies of the Software, and to permit persons to whom the\nSoftware is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included\nin all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS\nOR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING\nFROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER\nDEALINGS IN THE SOFTWARE.\n\n" }, + { + "name": "github.com/shopspring/decimal", + "path": "github.com/shopspring/decimal/LICENSE", + "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2015 Spring, Inc.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n\n- Based on https://github.com/oguzbilgic/fpd, which has the following license:\n\"\"\"\nThe MIT License (MIT)\n\nCopyright (c) 2013 Oguz Bilgic\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\"\"\"\n" + }, { "name": "github.com/sirupsen/logrus", "path": "github.com/sirupsen/logrus/LICENSE", @@ -989,6 +1039,16 @@ "path": "go.etcd.io/bbolt/LICENSE", "licenseText": "The MIT License (MIT)\n\nCopyright (c) 2013 Ben Johnson\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of\nthis software and associated documentation files (the \"Software\"), to deal in\nthe Software without restriction, including without limitation the rights to\nuse, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\nthe Software, and to permit persons to whom the Software is furnished to do so,\nsubject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\nFOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\nCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\nIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\nCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, + { + "name": "go.opentelemetry.io/otel", + "path": "go.opentelemetry.io/otel/LICENSE", + "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, + { + "name": "go.opentelemetry.io/otel/trace", + "path": "go.opentelemetry.io/otel/trace/LICENSE", + "licenseText": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" + }, { "name": "go.uber.org/atomic", "path": "go.uber.org/atomic/LICENSE.txt", diff --git a/build/generate-go-licenses.go b/build/generate-go-licenses.go index addab0762a5..c3b40c226f4 100644 --- a/build/generate-go-licenses.go +++ b/build/generate-go-licenses.go @@ -35,12 +35,40 @@ func main() { base, out := os.Args[1], os.Args[2] + // Add ext for excluded files because license_test.go will be included for some reason. + // And there are more files that should be excluded, check with: + // + // go run github.com/google/go-licenses@v1.6.0 save . --force --save_path=.go-licenses 2>/dev/null + // find .go-licenses -type f | while read FILE; do echo "${$(basename $FILE)##*.}"; done | sort -u + // AUTHORS + // COPYING + // LICENSE + // Makefile + // NOTICE + // gitignore + // go + // md + // mod + // sum + // toml + // txt + // yml + // + // It could be removed once we have a better regex. + excludedExt := map[string]bool{ + ".gitignore": true, + ".go": true, + ".mod": true, + ".sum": true, + ".toml": true, + ".yml": true, + } var paths []string err := filepath.WalkDir(base, func(path string, entry fs.DirEntry, err error) error { if err != nil { return err } - if entry.IsDir() || !licenseRe.MatchString(entry.Name()) { + if entry.IsDir() || !licenseRe.MatchString(entry.Name()) || excludedExt[filepath.Ext(entry.Name())] { return nil } paths = append(paths, path) diff --git a/cmd/actions.go b/cmd/actions.go index 66ad336da50..346de5b21a6 100644 --- a/cmd/actions.go +++ b/cmd/actions.go @@ -42,8 +42,7 @@ func runGenerateActionsRunnerToken(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) scope := c.String("scope") diff --git a/cmd/cmd.go b/cmd/cmd.go index 18d5db3987b..cf2d9ef89e8 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -57,8 +57,7 @@ func confirm() (bool, error) { } func initDB(ctx context.Context) error { - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) setting.LoadDBSetting() setting.InitSQLLog(false) diff --git a/cmd/doctor.go b/cmd/doctor.go index e7baad60c1f..65c028c5ed1 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -87,8 +87,7 @@ func runRecreateTable(ctx *cli.Context) error { golog.SetPrefix("") golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT))) - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) setting.LoadDBSetting() setting.Log.EnableXORMLog = ctx.Bool("debug") diff --git a/cmd/dump.go b/cmd/dump.go index 309bd01f664..32ccc5566c8 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -185,8 +185,7 @@ func runDump(ctx *cli.Context) error { } fileName += "." + outType } - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) // make sure we are logging to the console no matter what the configuration tells us do to // FIXME: don't use CfgProvider directly diff --git a/cmd/embedded.go b/cmd/embedded.go index cee8928ce08..3f849bea0a2 100644 --- a/cmd/embedded.go +++ b/cmd/embedded.go @@ -106,8 +106,9 @@ func initEmbeddedExtractor(c *cli.Context) error { log.DelNamedLogger(log.DEFAULT) // Read configuration file - setting.InitProviderAllowEmpty() - setting.LoadCommonSettings() + setting.Init(&setting.Options{ + AllowEmpty: true, + }) patterns, err := compileCollectPatterns(c.Args()) if err != nil { diff --git a/cmd/hook.go b/cmd/hook.go index 9605fcb3318..bd5575ab693 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/private" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "github.com/urfave/cli" ) @@ -141,7 +140,7 @@ func (d *delayWriter) Close() error { if d == nil { return nil } - stopped := util.StopTimer(d.timer) + stopped := d.timer.Stop() if stopped || d.buf == nil { return nil } diff --git a/cmd/mailer.go b/cmd/mailer.go index 50ba4b47411..74bae1ab68c 100644 --- a/cmd/mailer.go +++ b/cmd/mailer.go @@ -16,8 +16,7 @@ func runSendMail(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) if err := argsSet(c, "title"); err != nil { return err diff --git a/cmd/main_test.go b/cmd/main_test.go index ba323af4721..6e20be69451 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -7,14 +7,8 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/setting" ) -func init() { - setting.SetCustomPathAndConf("", "", "") - setting.InitProviderAndLoadCommonSettingsForTest() -} - func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ GiteaRootPath: "..", diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go index 887b59bba9e..5a7ede49397 100644 --- a/cmd/restore_repo.go +++ b/cmd/restore_repo.go @@ -51,8 +51,7 @@ func runRestoreRepository(c *cli.Context) error { ctx, cancel := installSignals() defer cancel() - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) var units []string if s := c.String("units"); s != "" { units = strings.Split(s, ",") diff --git a/cmd/serv.go b/cmd/serv.go index 72eb6370711..a79f314d00b 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -62,8 +62,7 @@ func setup(ctx context.Context, debug bool) { } else { _ = log.NewLogger(1000, "console", "console", `{"level":"fatal","stacktracelevel":"NONE","stderr":true}`) } - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) if debug { setting.RunMode = "dev" } diff --git a/cmd/web.go b/cmd/web.go index e451cf7dfa4..3a01d07b05f 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -177,8 +177,7 @@ func runWeb(ctx *cli.Context) error { log.Info("Global init") // Perform global initialization - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) routers.GlobalInitInstalled(graceful.GetManager().HammerContext()) // We check that AppDataPath exists here (it should have been created during installation) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index be043f33392..3ceb53dcd0a 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -303,7 +303,7 @@ RUN_MODE = ; prod LFS_JWT_SECRET = ;; ;; LFS authentication validity period (in time.Duration), pushes taking longer than this may fail. -;LFS_HTTP_AUTH_EXPIRY = 20m +;LFS_HTTP_AUTH_EXPIRY = 24h ;; ;; Maximum allowed LFS file size in bytes (Set to 0 for no limit). ;LFS_MAX_FILE_SIZE = 0 @@ -926,12 +926,6 @@ ROUTER = console ;; Global limit of repositories per user, applied at creation time. -1 means no limit ;MAX_CREATION_LIMIT = -1 ;; -;; Mirror sync queue length, increase if mirror syncing starts hanging (DEPRECATED: please use [queue.mirror] LENGTH instead) -;MIRROR_QUEUE_LENGTH = 1000 -;; -;; Patch test queue length, increase if pull request patch testing starts hanging (DEPRECATED: please use [queue.pr_patch_checker] LENGTH instead) -;PULL_REQUEST_QUEUE_LENGTH = 1000 -;; ;; Preferred Licenses to place at the top of the List ;; The name here must match the filename in options/license or custom/options/license ;PREFERRED_LICENSES = Apache License 2.0,MIT License @@ -1376,22 +1370,6 @@ ROUTER = console ;; Set to -1 to disable timeout. ;STARTUP_TIMEOUT = 30s ;; -;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer]) -;ISSUE_INDEXER_QUEUE_TYPE = levelqueue; **DEPRECATED** use settings in `[queue.issue_indexer]`. -;; -;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved. -;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`. -;; default is queues/common -;ISSUE_INDEXER_QUEUE_DIR = queues/common; **DEPRECATED** use settings in `[queue.issue_indexer]`. Relative paths will be made absolute against `%(APP_DATA_PATH)s`. -;; -;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. -;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of -;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. -;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"; **DEPRECATED** use settings in `[queue.issue_indexer]`. -;; -;; Batch queue number, default is 20 -;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`. - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Repository Indexer settings ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1418,8 +1396,6 @@ ROUTER = console ;; A comma separated list of glob patterns to exclude from the index; ; default is empty ;REPO_INDEXER_EXCLUDE = ;; -;; -;UPDATE_BUFFER_LEN = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`. ;MAX_FILE_SIZE = 1048576 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1441,7 +1417,7 @@ ROUTER = console ;DATADIR = queues/ ; Relative paths will be made absolute against `%(APP_DATA_PATH)s`. ;; ;; Default queue length before a channel queue will block -;LENGTH = 20 +;LENGTH = 100 ;; ;; Batch size to send for batched queues ;BATCH_LENGTH = 20 @@ -1449,7 +1425,7 @@ ROUTER = console ;; Connection string for redis queues this will store the redis connection string. ;; When `TYPE` is `persistable-channel`, this provides a directory for the underlying leveldb ;; or additional options of the form `leveldb://path/to/db?option=value&....`, and will override `DATADIR`. -;CONN_STR = "addrs=127.0.0.1:6379 db=0" +;CONN_STR = "redis://127.0.0.1:6379/0" ;; ;; Provides the suffix of the default redis/disk queue name - specific queues can be overridden within in their [queue.name] sections. ;QUEUE_NAME = "_queue" @@ -1457,29 +1433,8 @@ ROUTER = console ;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections. ;SET_NAME = "_unique" ;; -;; If the queue cannot be created at startup - level queues may need a timeout at startup - wrap the queue: -;WRAP_IF_NECESSARY = true -;; -;; Attempt to create the wrapped queue at max -;MAX_ATTEMPTS = 10 -;; -;; Timeout queue creation -;TIMEOUT = 15m30s -;; -;; Create a pool with this many workers -;WORKERS = 0 -;; ;; Dynamically scale the worker pool to at this many workers ;MAX_WORKERS = 10 -;; -;; Add boost workers when the queue blocks for BLOCK_TIMEOUT -;BLOCK_TIMEOUT = 1s -;; -;; Remove the boost workers after BOOST_TIMEOUT -;BOOST_TIMEOUT = 5m -;; -;; During a boost add BOOST_WORKERS -;BOOST_WORKERS = 1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -2512,6 +2467,8 @@ ROUTER = console ;LIMIT_SIZE_PUB = -1 ;; Maximum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_PYPI = -1 +;; Maximum size of a RPM upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +;LIMIT_SIZE_RPM = -1 ;; Maximum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) ;LIMIT_SIZE_RUBYGEMS = -1 ;; Maximum size of a Swift upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/assets/js/search.js b/docs/assets/js/search.js index f731e0b224c..4b95b638472 100644 --- a/docs/assets/js/search.js +++ b/docs/assets/js/search.js @@ -53,7 +53,7 @@ function doSearch() { } else { const para = document.createElement('P'); para.textContent = 'Please enter a word or phrase above'; - document.getElementById('search-results').appendChild(para); + document.getElementById('search-results').append(para); } } @@ -85,7 +85,7 @@ function executeSearch(searchQuery) { } else { const para = document.createElement('P'); para.textContent = 'No matches found'; - document.getElementById('search-results').appendChild(para); + document.getElementById('search-results').append(para); } }); } @@ -128,7 +128,7 @@ function populateResults(result) { categories: value.item.categories, snippet }); - document.getElementById('search-results').appendChild(htmlToElement(output)); + document.getElementById('search-results').append(htmlToElement(output)); for (const snipvalue of snippetHighlights) { new Mark(document.getElementById(`summary-${key}`)).mark(snipvalue); diff --git a/docs/content/doc/actions.en-us.md b/docs/content/doc/actions.en-us.md new file mode 100644 index 00000000000..7cd2ba0546f --- /dev/null +++ b/docs/content/doc/actions.en-us.md @@ -0,0 +1,13 @@ +--- +date: "2023-04-27T14:00:00+08:00" +title: "Actions" +slug: "actions" +weight: 36 +toc: false +draft: false +menu: + sidebar: + name: "Usage - Actions" + weight: 31 + identifier: "actions" +--- diff --git a/docs/content/doc/actions/act-runner.en-us.md b/docs/content/doc/actions/act-runner.en-us.md new file mode 100644 index 00000000000..03d01bfa7cd --- /dev/null +++ b/docs/content/doc/actions/act-runner.en-us.md @@ -0,0 +1,206 @@ +--- +date: "2023-04-27T15:00:00+08:00" +title: "Act Runner" +slug: "usage/actions/act-runner" +weight: 20 +draft: false +toc: false +menu: + sidebar: + parent: "actions" + name: "Act Runner" + weight: 20 + identifier: "actions-runner" +--- + +# Act Runner + +This page will introduce the [act runner](https://gitea.com/gitea/act_runner) in detail, which is the runner of Gitea Actions. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +It is recommended to run jobs in a docker container, so you need to install docker first. +And make sure that the docker daemon is running. + +Other OCI container engines which are compatible with Docker's API should also work, but are untested. + +However, if you are sure that you want to run jobs directly on the host only, then docker is not required. + +## Installation + +There are multiple ways to install the act runner. + +### Download the binary + +You can download the binary from the [release page](https://gitea.com/gitea/act_runner/releases). +However, if you want to use the latest nightly build, you can download it from the [download page](https://dl.gitea.com/act_runner/). + +When you download the binary, please make sure that you have downloaded the correct one for your platform. +You can check it by running the following command: + +```bash +chmod +x act_runner +./act_runner --version +``` + +If you see the version information, it means that you have downloaded the correct binary. + +### Use the docker image + +You can use the docker image from the [docker hub](https://hub.docker.com/r/gitea/act_runner/tags). +Just like the binary, you can use the latest nightly build by using the `nightly` tag, while the `latest` tag is the latest stable release. + +```bash +docker pull gitea/act_runner:latest # for the latest stable release +docker pull gitea/act_runner:nightly # for the latest nightly build +``` + +## Configuration + +Configuration is done via a configuration file. It is optional, and the default configuration will be used when no configuration file is specified. + +You can generate a configuration file by running the following command: + +```bash +./act_runner generate-config +``` + +The default configuration is safe to use without any modification, so you can just use it directly. + +```bash +./act_runner generate-config > config.yaml +./act_runner --config config.yaml [command] +``` + +When you are using the docker image, you can specify the configuration file by using the `CONFIG_FILE` environment variable. Make sure that the file is mounted into the container as a volume: + +```bash +docker run -v $(pwd)/config.yaml:/config.yaml -e CONFIG_FILE=/config.yaml ... +``` + +You may notice the commands above are both incomplete, because it is not the time to run the act runner yet. +Before running the act runner, we need to register it to your Gitea instance first. + +## Registration + +Registration is required before running the act runner, because the runner needs to know where to get jobs from. +And it is also important to Gitea instance to identify the runner. + +### Runner levels + +You can register a runner in different levels, it can be: + +- Instance level: The runner will run jobs for all repositories in the instance. +- Organization level: The runner will run jobs for all repositories in the organization. +- Repository level: The runner will run jobs for the repository it belongs to. + +Note that the repository may still use instance-level or organization-level runners even if it has its own repository-level runners. A future release may provide an option to allow more control over this. + +### Obtain a registration token + +The level of the runner determines where to obtain the registration token. + +- Instance level: The admin settings page, like `/admin/runners`. +- Organization level: The organization settings page, like `//settings/runners`. +- Repository level: The repository settings page, like `///settings/runners`. + +If you cannot see the settings page, please make sure that you have the right permissions and that Actions have been enabled. + +The format of the registration token is a random string `D0gvfu2iHfUjNqCYVljVyRV14fISpJxxxxxxxxxx`. + +### Register the runner + +The act runner can be registered by running the following command: + +```bash +./act_runner register +``` + +Alternatively, you can use the `--config` option to specify the configuration file mentioned in the previous section. + +```bash +./act_runner --config config.yaml register +``` + +You will be asked to input the registration information step by step. Includes: + +- The Gitea instance URL, like `https://gitea.com/` or `http://192.168.8.8:3000/`. +- The registration token. +- The runner name, which is optional. If you leave it blank, the hostname will be used. +- The runner labels, which is optional. If you leave it blank, the default labels will be used. + +You may be confused about the runner labels, which will be explained later. + +If you want to register the runner in a non-interactive way, you can use arguments to do it. + +```bash +./act_runner register --no-interactive --instance --token --name --labels +``` + +When you have registered the runner, you can find a new file named `.runner` in the current directory. +This file stores the registration information. +Please do not edit it manually. +If this file is missing or corrupted, you can simply remove it and register again. + +If you want to store the registration information in another place, you can specify it in the configuration file, +and don't forget to specify the `--config` option. + +### Register the runner with docker + +If you are using the docker image, behaviour will be slightly different. Registration and running are combined into one step in this case, so you need to specify the registration information when running the act runner. + +```bash +docker run \ + -v $(pwd)/config.yaml:/config.yaml \ + -v $(pwd)/data:/data \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e CONFIG_FILE=/config.yaml \ + -e GITEA_INSTANCE_URL= \ + -e GITEA_RUNNER_REGISTRATION_TOKEN= \ + -e GITEA_RUNNER_NAME= \ + -e GITEA_RUNNER_LABELS= \ + --name my_runner \ + -d gitea/act_runner:nightly +``` + +You may notice that we have mounted the `/var/run/docker.sock` into the container. +It is because the act runner will run jobs in docker containers, so it needs to communicate with the docker daemon. +As mentioned, you can remove it if you want to run jobs in the host directly. +To be clear, the "host" actually means the container which is running the act runner now, instead of the host machine. + +### Labels + +The labels of a runner are used to determine which jobs the runner can run, and how to run them. + +The default labels are `ubuntu-latest:docker://node:16-bullseye,ubuntu-22.04:docker://node:16-bullseye,ubuntu-20.04:docker://node:16-bullseye,ubuntu-18.04:docker://node:16-buster`. +It is a comma-separated list, and each item is a label. + +Let's take `ubuntu-22.04:docker://node:16-bullseye` as an example. +It means that the runner can run jobs with `runs-on: ubuntu-22.04`, and the job will be run in a docker container with the image `node:16-bullseye`. + +If the default image is insufficient for your needs, and you have enough disk space to use a better and bigger one, you can change it to `ubuntu-22.04:docker://`. +You can find more useful images on [act images](https://github.com/nektos/act/blob/master/IMAGES.md). + +If you want to run jobs in the host directly, you can change it to `ubuntu-22.04:host` or just `ubuntu-22.04`, the `:host` is optional. +However, we suggest you to use a special name like `linux_amd64:host` or `windows:host` to avoid misusing it. + +One more thing is that it is recommended to register the runner if you want to change the labels. +It may be annoying to do this, so we may provide a better way to do it in the future. + +## Running + +After you have registered the runner, you can run it by running the following command: + +```bash +./act_runner daemon +# or +./act_runner daemon --config config.yaml +``` + +The runner will fetch jobs from the Gitea instance and run them automatically. + +Since act runner is still in development, it is recommended to check the latest version and upgrade it regularly. diff --git a/docs/content/doc/actions/comparison.en-us.md b/docs/content/doc/actions/comparison.en-us.md new file mode 100644 index 00000000000..209fa2adc0b --- /dev/null +++ b/docs/content/doc/actions/comparison.en-us.md @@ -0,0 +1,171 @@ +--- +date: "2023-04-27T15:00:00+08:00" +title: "Compared to GitHub Actions" +slug: "usage/actions/comparison" +weight: 30 +draft: false +toc: false +menu: + sidebar: + parent: "actions" + name: "Comparison" + weight: 30 + identifier: "actions-comparison" +--- + +# Compared to GitHub Actions + +Even though Gitea Actions is designed to be compatible with GitHub Actions, there are some differences between them. + +**Table of Contents** + +{{< toc >}} + +## Additional features + +### Absolute action URLs + +Gitea Actions supports defining actions via absolute URL, which means that you can use actions from any git repository. +Like `uses: https://github.com/actions/checkout@v3` or `uses: http://your_gitea.com/owner/repo@branch`. + +### Actions written in Go + +Gitea Actions supports writing actions in Go. +See [Creating Go Actions](https://blog.gitea.io/2023/04/creating-go-actions/). + +## Unsupported workflows syntax + +### `concurrency` + +It's used to run a single job at a time. +See [Using concurrency](https://docs.github.com/en/actions/using-jobs/using-concurrency). + +It's ignored by Gitea Actions now. + +### `run-name` + +The name for workflow runs generated from the workflow. +See [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#run-name). + +It's ignored by Gitea Actions now. + +### `permissions` and `jobs..permissions` + +See [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions). + +It's ignored by Gitea Actions now. + +### `jobs..timeout-minutes` + +See [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idtimeout-minutes). + +It's ignored by Gitea Actions now. + +### `jobs..continue-on-error` + +See [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idcontinue-on-error). + +It's ignored by Gitea Actions now. + +### `jobs..environment` + +See [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idenvironment). + +It's ignored by Gitea Actions now. + +### Complex `runs-on` + +See [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on). + +Gitea Actions only supports `runs-on: xyz` or `runs-on: [xyz]` now. + +### `workflow_dispatch` + +See [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#onworkflow_dispatch). + +It's ignored by Gitea Actions now. + +### `hashFiles` expression + +See [Expressions](https://docs.github.com/en/actions/learn-github-actions/expressions#hashfiles) + +Gitea Actions doesn't support it now, if you use it, the result will always be empty string. + +As a workaround, you can use [go-hashfiles](https://gitea.com/actions/go-hashfiles) instead. + +## Missing features + +### Variables + +See [Variables](https://docs.github.com/en/actions/learn-github-actions/variables). + +It's under development. + +### Problem Matchers + +Problem Matchers are a way to scan the output of actions for a specified regex pattern and surface that information prominently in the UI. +See [Problem matchers](https://github.com/actions/toolkit/blob/main/docs/problem-matchers.md). + +It's ignored by Gitea Actions now. + +### Create an error annotation + +See [Creating an annotation for an error](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#example-creating-an-annotation-for-an-error) + +It's ignored by Gitea Actions now. + +## Missing UI features + +### Pre and Post steps + +Pre and Post steps don't have their own section in the job log user interface. + +## Different behavior + +### Downloading actions + +Gitea Actions doesn't download actions from GitHub by default. +"By default" means that you don't specify the host in the `uses` field, like `uses: actions/checkout@v3`. +As a contrast, `uses: https://github.com/actions/checkout@v3` has specified host. + +The missing host will be filled with `https://gitea.com` if you don't configure it. +That means `uses: actions/checkout@v3` will download the action from [gitea.com/actions/checkout](https://gitea.com/actions/checkout), instead of [github.com/actions/checkout](https://github.com/actions/checkout). + +As mentioned, it's configurable. +If you want your runners to download actions from GitHub or your own Gitea instance by default, you can configure it by setting `[actions].DEFAULT_ACTIONS_URL`. See [Configuration Cheat Sheet]({{< relref "doc/administration/config-cheat-sheet.en-us.md#actions-actions" >}}). + +### Context availability + +Context availability is not checked, so you can use the env context on more places. +See [Context availability](https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability). + +## Known issues + +### `docker/build-push-action@v4` + +See [act_runner#119](https://gitea.com/gitea/act_runner/issues/119#issuecomment-738294). + +`ACTIONS_RUNTIME_TOKEN` is a random string in Gitea Actions, not a JWT. +But the `docker/build-push-action@v4` tries to parse the token as JWT and doesn't handle the error, so the job fails. + +There are two workarounds: + +Set the `ACTIONS_RUNTIME_TOKEN` to empty manually, like: + +``` yml +- name: Build and push + uses: docker/build-push-action@v4 + env: + ACTIONS_RUNTIME_TOKEN: '' + with: +... +``` + +The bug has been fixed in a newer [commit](https://gitea.com/docker/build-push-action/commit/d8823bfaed2a82c6f5d4799a2f8e86173c461aba?style=split&whitespace=show-all#diff-1af9a5bdf96ddff3a2f3427ed520b7005e9564ad), but it has not been released. So you could use the latest version by specifying the branch name, like: + +``` yml +- name: Build and push + uses: docker/build-push-action@master + with: +... +``` diff --git a/docs/content/doc/actions/design.en-us.md b/docs/content/doc/actions/design.en-us.md new file mode 100644 index 00000000000..45764079f88 --- /dev/null +++ b/docs/content/doc/actions/design.en-us.md @@ -0,0 +1,135 @@ +--- +date: "2023-04-27T15:00:00+08:00" +title: "Design of Gitea Actions" +slug: "usage/actions/design" +weight: 40 +draft: false +toc: false +menu: + sidebar: + parent: "actions" + name: "Design" + weight: 40 + identifier: "actions-design" +--- + +# Design of Gitea Actions + +Gitea Actions has multiple components. This document describes them individually. + +**Table of Contents** + +{{< toc >}} + +## Act + +The [nektos/act](https://github.com/nektos/act) project is an excellent tool that allows you to run your GitHub Actions locally. +We were inspired by this and wondered if it would be possible to run actions for Gitea. + +However, while [nektos/act](https://github.com/nektos/act) is designed as a command line tool, what we actually needed was a Go library with modifications specifically for Gitea. +So we forked it as [gitea/act](https://gitea.com/gitea/act). + +This is a soft fork that will periodically follow the upstream. +Although some custom commits have been added, we will try our best to avoid changing too much of the original code. + +The forked act is just a shim or adapter for Gitea's specific usage. +There are some additional commits that have been made, such as: + +- Outputting execution logs to logger hook so they can be reported to Gitea +- Disabling the GraphQL URL, since Gitea doesn't support it +- Starting a new container for every job instead of reusing to ensure isolation. + +These modifications have no reason to be merged into the upstream. +They don't make sense if the user just wants to run trusted actions locally. + +However, there may be overlaps in the future, such as a required bug fix or new feature needed by both projects. +In these cases, we will contribute the changes back to the upstream repository. + +## Act runner + +Gitea's runner is called act runner because it's based on act. + +Like other CI runners, we designed it as an external part of Gitea, which means it should run on a different server than Gitea. + +To ensure that the runner connects to the correct Gitea instance, we need to register it with a token. +Additionally, the runner will introduce itself to Gitea and declare what kind of jobs it can run by reporting its labels. + +Earlier, we mentioned that `runs-on: ubuntu-latest` in a workflow file means that the job will be run on a runner with the `ubuntu-latest` label. +But how does the runner know to run `ubuntu-latest`? The answer lies in mapping the label to an environment. +That's why when you add custom labels during registration, you will need to input some complex content like `my_custom_label:docker://centos:7`. +This means that the runner can take the job which needs to run on `my_custom_label`, and it will run it via a docker container with the image `centos:7`. + +Docker isn't the only option, though. +The act also supports running jobs directly on the host. +This is achieved through labels like `linux_arm:host`. +This label indicates that the runner can take a job that needs to run on `linux_arm` and run it directly on the host. + +The label's design follows the format `label[:schema[:args]]`. +If the schema is omitted, it defaults to `host`. +So, + +- `my_custom_label:docker://node:18`: Run jobs labeled with `my_custom_label` using the `node:18` Docker image. +- `my_custom_label:host`: Run jobs labeled with `my_custom_label` directly on the host. +- `my_custom_label`: Same as `my_custom_label:host`. +- `my_custom_label:vm:ubuntu-latest`: (Example only, not implemented) Run jobs labeled with `my_custom_label` using a virtual machine with the `ubuntu-latest` ISO. + +## Communication protocol + +As act runner is an independent part of Gitea, we needed a protocol for runners to communicate with the Gitea instance. +However, we did not think it was a good idea to have Gitea listen on a new port. +Instead, we wanted to reuse the HTTP port, which means we needed a protocol that is compatible with HTTP. +We chose to use gRPC over HTTP. + +We use [actions-proto-def](https://gitea.com/gitea/actions-proto-def) and [actions-proto-go](https://gitea.com/gitea/actions-proto-go) to wire them up. +More information about gRPC can be found on [its website](https://grpc.io/). + +## Network architecture + +Let's examine the overall network architecture. +This will help you troubleshoot some problems and explain why it's a bad idea to register a runner with a loopback address of the Gitea instance. + +![network](/images/usage/actions/network.png) + +There are four network connections marked in the picture, and the direction of the arrows indicates the direction of establishing the connections. + +### Connection 1, act runner to Gitea instance + +The act runner must be able to connect to Gitea to receive tasks and send back the execution results. + +### Connection 2, job containers to Gitea instance + +The job containers have different network namespaces than the runner, even if they are on the same machine. +They need to connect to Gitea to fetch codes if there is `actions/checkout@v3` in the workflow, for example. +Fetching code is not always necessary to run some jobs, but it is required in most cases. + +If you use a loopback address to register a runner, the runner can connect to Gitea when it is on the same machine. +However, if a job container tries to fetch code from localhost, it will fail because Gitea is not in the same container. + +### Connection 3, act runner to internet + +When you use some actions like `actions/checkout@v3`, the act runner downloads the scripts, not the job containers. +By default, it downloads from [gitea.com](http://gitea.com/), so it requires access to the internet. +It also downloads some docker images from Docker Hub by default, which also requires internet access. + +However, internet access is not strictly necessary. +You can configure your Gitea instance to fetch actions or images from your intranet facilities. + +In fact, your Gitea instance can serve as both the actions marketplace and the image registry. +You can mirror actions repositories from GitHub to your Gitea instance, and use them as normal. +And [Gitea Container Registry](https://docs.gitea.io/en-us/usage/packages/container/) can be used as a Docker image registry. + +### Connection 4, job containers to internet + +When using actions such as `actions/setup-go@v4`, it may be necessary to download resources from the internet to set up the Go language environment in job containers. +Therefore, access to the internet is required for the successful completion of these actions. + +However, it is optional as well. +You can use your own custom actions to avoid relying on internet access, or you can use your packaged Docker image to run jobs with all dependencies installed. + +## Summary + +Using Gitea Actions only requires ensuring that the runner can connect to the Gitea instance. +Internet access is optional, but not having it will require some additional work. +In other words: The runner works best when it can query the internet itself, but you don't need to expose it to the internet (in either direction). + +If you encounter any network issues while using Gitea Actions, hopefully the image above can help you troubleshoot them. diff --git a/docs/content/doc/actions/faq.en-us.md b/docs/content/doc/actions/faq.en-us.md new file mode 100644 index 00000000000..bec41db3ea4 --- /dev/null +++ b/docs/content/doc/actions/faq.en-us.md @@ -0,0 +1,166 @@ +--- +date: "2023-04-27T15:00:00+08:00" +title: "Frequently Asked Questions of Gitea Actions" +slug: "usage/actions/faq" +weight: 100 +draft: false +toc: false +menu: + sidebar: + parent: "actions" + name: "FAQ" + weight: 100 + identifier: "actions-faq" +--- + +# Frequently Asked Questions of Gitea Actions + +This page contains some common questions and answers about Gitea Actions. + +**Table of Contents** + +{{< toc >}} + +## Why is Actions not enabled by default? + +We know it's annoying to enable Actions for the whole instance and each repository one by one, but not everyone likes or needs this feature. +We believe that more work needs to be done to improve Gitea Actions before it deserves any further special treatment. + +## Is it possible to enable Actions for new repositories by default for my own instance? + +Yes, when you enable Actions for the instance, you can choose to enable the `actions` unit for all new repositories by default. + +```ini +[repository] +DEFAULT_REPO_UNITS = ...,repo.actions +``` + +## Should we use `${{ github.xyz }}` or `${{ gitea.xyz }}` in workflow files? + +You can use `github.xyz` and Gitea will work fine. +As mentioned, Gitea Actions is designed to be compatible with GitHub Actions. +However, we recommend using `gitea.xyz` in case Gitea adds something that GitHub does not have to avoid different kinds of secrets in your workflow file (and because you are using this workflow on Gitea, not GitHub). +Still, this is completely optional since both options have the same effect at the moment. + +## Is it possible to register runners for a specific user (not organization)? + +Not yet. +It is technically possible to implement, but we need to discuss whether it is necessary. + +## Where will the runner download scripts when using actions such as `actions/checkout@v3`? + +You may be aware that there are tens of thousands of [marketplace actions](https://github.com/marketplace?type=actions) in GitHub. +However, when you write `uses: actions/checkout@v3`, it actually downloads the scripts from [gitea.com/actions/checkout](http://gitea.com/actions/checkout) by default (not GitHub). +This is a mirror of [github.com/actions/checkout](http://github.com/actions/checkout), but it's impossible to mirror all of them. +That's why you may encounter failures when trying to use some actions that haven't been mirrored. + +The good news is that you can specify the URL prefix to use actions from anywhere. +This is an extra syntax in Gitea Actions. +For example: + +- `uses: https://github.com/xxx/xxx@xxx` +- `uses: https://gitea.com/xxx/xxx@xxx` +- `uses: http://your_gitea_instance.com/xxx@xxx` + +Be careful, the `https://` or `http://` prefix is necessary! + +Alternatively, if you want your runners to download actions from GitHub or your own Gitea instance by default, you can configure it by setting `[actions].DEFAULT_ACTIONS_URL`. +See [Configuration Cheat Sheet](https://docs.gitea.io/en-us/config-cheat-sheet/#actions-actions). + +This is one of the differences from GitHub Actions, but it should allow users much more flexibility in how they run Actions. + +## How to limit the permission of the runners? + +Runners have no more permissions than simply connecting to your Gitea instance. +When any runner receives a job to run, it will temporarily gain limited permission to the repository associated with the job. +If you want to give more permissions to the runner, allowing it to access more private repositories or external systems, you can pass [secrets](https://docs.gitea.io/en-us/usage/secrets/) to it. + +Refined permission control to Actions is a complicated job. +In the future, we will add more options to Gitea to make it more configurable, such as allowing more write access to repositories or read access to all repositories in the same organization. + +## How to avoid being hacked? + +There are two types of possible attacks: unknown runner stealing the code or secrets from your repository, or malicious scripts controlling your runner. + +Avoiding the former means not allowing people you don't know to register runners for your repository, organization, or instance. + +The latter is a bit more complicated. +If you're using a private Gitea instance for your company, you may not need to worry about security since you trust your colleagues and can hold them accountable. + +For public instances, things are a little different. +Here's how we do it on [gitea.com](http://gitea.com/): + +- We only register runners for the "gitea" organization, so our runners will not execute jobs from other repositories. +- Our runners always run jobs with isolated containers. While it is possible to do this directly on the host, we choose not to for more security. +- To run actions for fork pull requests, approval is required. See [#22803](https://github.com/go-gitea/gitea/pull/22803). +- If someone registers their own runner for their repository or organization on [gitea.com](http://gitea.com/), we have no objections and will just not use it in our org. However, they should take care to ensure that the runner is not used by other users they do not know. + +## Which operating systems are supported by act runner? + +It works well on Linux, macOS, and Windows. +While other operating systems are theoretically supported, they require further testing. + +One thing to note is that if you choose to run jobs directly on the host instead of in job containers, the environmental differences between operating systems may cause unexpected failures. + +For example, bash is not available on Windows in most cases, while act tries to use bash to run scripts by default. +Therefore, you need to specify `powershell` as the default shell in your workflow file, see [defaults.run](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun). + +```yaml +defaults: + run: + shell: powershell +``` + +## Why choose GitHub Actions? Why not something compatible with GitLab CI/CD? + +[@lunny](https://gitea.com/lunny) has explained this in the [issue to implement actions](https://github.com/go-gitea/gitea/issues/13539). +Furthermore, Actions is not only a CI/CD system but also an automation tool. + +There have also been numerous [marketplace actions](https://github.com/marketplace?type=actions) implemented in the open-source world. +It is exciting to be able to reuse them. + +## What if it runs on multiple labels, such as `runs-on: [label_a, label_b]`? + +This is valid syntax. +It means that it should run on runners that have both the `label_a` **and** `label_b` labels, see [Workflow syntax for GitHub Actions](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idruns-on). +Unfortunately, act runner does not work this way. +As mentioned, we map labels to environments: + +- `ubuntu` → `ubuntu:22.04` +- `centos` → `centos:8` + +But we need to map label groups to environments instead, like so: + +- `[ubuntu]` → `ubuntu:22.04` +- `[with-gpu]` → `linux:with-gpu` +- `[ubuntu, with-gpu]` → `ubuntu:22.04_with-gpu` + +We also need to re-design how tasks are assigned to runners. +A runner with `ubuntu`, `centos`, or `with-gpu` does not necessarily indicate that it can accept jobs with `[centos, with-gpu]`. +Therefore, the runner should inform the Gitea instance that it can only accept jobs with `[ubuntu]`, `[centos]`, `[with-gpu]`, and `[ubuntu, with-gpu]`. +This is not a technical problem, it was just overlooked in the early design. +See [runtime.go#L65](https://gitea.com/gitea/act_runner/src/commit/90b8cc6a7a48f45cc28b5ef9660ebf4061fcb336/runtime/runtime.go#L65). + +Currently, the act runner attempts to match everyone in the labels and uses the first match it finds. + +## What is the difference between agent labels and custom labels for a runner? + +![labels](/images/usage/actions/labels.png) + +Agent labels are reported to the Gitea instance by the runner during registration. +Custom labels, on the other hand, are added manually by a Gitea administrator or owners of the organization or repository (depending on the level of the runner). + +However, the design here needs improvement, as it currently has some rough edges. +You can add a custom label such as `centos` to a registered runner, which means the runner will receive jobs with `runs-on: centos`. +However, the runner may not know which environment to use for this label, resulting in it using a default image or leading to a logical dead end. +This default may not match user expectations. +See [runtime.go#L71](https://gitea.com/gitea/act_runner/src/commit/90b8cc6a7a48f45cc28b5ef9660ebf4061fcb336/runtime/runtime.go#L71). + +In the meantime, we suggest that you re-register your runner if you want to change its labels. + +## Will there be more implementations for Gitea Actions runner? + +Although we would like to provide more options, our limited manpower means that act runner will be the only officially supported runner. +However, both Gitea and act runner are completely open source, so anyone can create a new/better implementation. +We support your choice, no matter how you decide. +In case you fork act runner to create your own version: Please contribute the changes back if you can and if you think your changes will help others as well. diff --git a/docs/content/doc/actions/overview.en-us.md b/docs/content/doc/actions/overview.en-us.md new file mode 100644 index 00000000000..ef52465e25d --- /dev/null +++ b/docs/content/doc/actions/overview.en-us.md @@ -0,0 +1,55 @@ +--- +date: "2023-04-27T15:00:00+08:00" +title: "Gitea Actions" +slug: "usage/actions/overview" +weight: 1 +draft: false +toc: false +menu: + sidebar: + parent: "actions" + name: "Overview" + weight: 1 + identifier: "actions-overview" +--- + +# Gitea Actions + +Starting with Gitea **1.19**, Gitea Actions are available as a built-in CI/CD solution. + +**Table of Contents** + +{{< toc >}} + +## Name + +It is similar and compatible to [GitHub Actions](https://github.com/features/actions), and its name is inspired by it too. +To avoid confusion, we have clarified the spelling here: + +- "Gitea Actions" (with an "s", both words capitalized) is the name of the Gitea feature. +- "GitHub Actions" is the name of the GitHub feature. +- "Actions" could refer to either of the above, depending on the context. So it refers to "Gitea Actions" in this document. +- "action" or "actions" refer to some scripts/plugins to be used, like "actions/checkout@v3" or "actions/cache@v3". + +## Runners + +Just like other CI/CD solutions, Gitea doesn't run the jobs itself, but delegates the jobs to runners. +The runner of Gitea Actions is called [act runner](https://gitea.com/gitea/act_runner), it is a standalone program and also written in Go. +It is based on a [fork](https://gitea.com/gitea/act) of [nektos/act](http://github.com/nektos/act). + +Because the runner is deployed independently, there could be potential security issues. +To avoid them, please follow two simple rules: + +- Don't use a runner you don't trust for your repository, organization or instance. +- Don't provide a runner to a repository, organization or instance you don't trust. + +For Gitea instances used internally, such as instances used by enterprises or individuals, neither of these two rules is a problem, they are naturally so. +However, for public Gitea instances, such as [gitea.com](https://gitea.com), these two rules should be kept in mind when adding or using runners. + +## Status + +Gitea Actions is still under development, so there may be some bugs and missing features. +And breaking changes may be made before it's stable (v1.20 or later). + +If the situation changes, we will update it here. +So please refer to the content here when you find outdated articles elsewhere. diff --git a/docs/content/doc/actions/quickstart.en-us.md b/docs/content/doc/actions/quickstart.en-us.md new file mode 100644 index 00000000000..72f1dda0829 --- /dev/null +++ b/docs/content/doc/actions/quickstart.en-us.md @@ -0,0 +1,141 @@ +--- +date: "2023-04-27T15:00:00+08:00" +title: "Quick Start" +slug: "usage/actions/quickstart" +weight: 10 +draft: false +toc: false +menu: + sidebar: + parent: "actions" + name: "Quick Start" + weight: 10 + identifier: "actions-quickstart" +--- + +# Quick Start + +This page will guide you through the process of using Gitea Actions. + +**Table of Contents** + +{{< toc >}} + +## Set up Gitea + +First of all, you need a Gitea instance. +You can follow the [documentation]({{< relref "doc/installation/from-package.en-us.md" >}}) to set up a new instance or upgrade your existing one. +It doesn't matter how you install or run Gitea, as long as its version is 1.19.0 or higher. + +Actions are disabled by default, so you need to add the following to the configuration file to enable it: + +```ini +[actions] +ENABLED=true +``` + +If you want to learn more or encounter any problems while configuring it, please refer to the [Configuration Cheat Sheet]({{< relref "doc/administration/config-cheat-sheet.en-us.md#actions-actions" >}}). + +### Set up runner + +Gitea Actions requires [act runner](https://gitea.com/gitea/act_runner) to run the jobs. +In order to avoid consuming too many resources and affecting the Gitea instance, it is recommended to start runners on separate machines from the Gitea instance. + +You can use the [pre-built binaries](http://dl.gitea.com/act_runner) or the [docker images](https://hub.docker.com/r/gitea/act_runner/tags) to set up the runner. + +Before proceeding any further, we suggest running it as a command line with pre-built binaries to ensure that it works with your environment, especially if you are running a runner on your local host. +And it could be easier to debug if something goes wrong. + +The runner can run the jobs in isolated Docker containers, so you need to make sure that the Docker has been installed and Docker daemon is running. +While it is not strictly necessary, because the runner can also run the jobs directly on the host, it depends on how you configure it. +However, it is recommended to use Docker to run the jobs, because it is more secure and easier to manage. + +Before running a runner, you should first register it to your Gitea instance using the following command: + +```bash +./act_runner register --no-interactive --instance --token +``` + +There are two arguments required, `instance` and `token`. + +`instance` refers to the address of your Gitea instance, like `http://192.168.8.8:3000` or `https://gitea.com`. +The runner and job containers (which are started by the runner to execute jobs) will connect to this address. +This means that it could be different from the `ROOT_URL` of your Gitea instance, which is configured for web access. +It is always a bad idea to use a loopback address such as `127.0.0.1` or `localhost`. +If you are unsure which address to use, the LAN address is usually the right choice. + +`token` is used for authentication and identification, such as `P2U1U0oB4XaRCi8azcngmPCLbRpUGapalhmddh23`. +It is one-time use only and cannot be used to register multiple runners. +You can obtain tokens from `/admin/runners`. + +![register runner](/images/usage/actions/register-runner.png) + +After registering, a new file named `.runner` will appear in the current directory. +This file stores the registration information. +Please do not edit it manually. +If this file is missing or corrupted, you can simply remove it and register again. + +Finally, it's time to start the runner: + +```bash +./act_runner daemon +``` + +And you can see the new runner in the management page: + +![view runner](/images/usage/actions/view-runner.png) + +You can find more information by visiting [Act runner]({{< relref "doc/actions/act-runner.en-us.md" >}}). + +### Use Actions + +Even if Actions is enabled for the Gitea instance, repositories still disable Actions by default. + +To enable it, go to the settings page of your repository like `your_gitea.com//repo/settings` and enable `Enable Repository Actions`. + +![enable actions](/images/usage/actions/enable-actions.png) + +The next steps may be rather complicated. +You will need to study [the workflow syntax](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions) for Actions and write the workflow files you want. + +However, we can just start from a simple demo: + +```yaml +name: Gitea Actions Demo +run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀 +on: [push] + +jobs: + Explore-Gitea-Actions: + runs-on: ubuntu-latest + steps: + - run: echo "🎉 The job was automatically triggered by a ${{ gitea.event_name }} event." + - run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by Gitea!" + - run: echo "🔎 The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}." + - name: Check out repository code + uses: actions/checkout@v3 + - run: echo "💡 The ${{ gitea.repository }} repository has been cloned to the runner." + - run: echo "🖥️ The workflow is now ready to test your code on the runner." + - name: List files in the repository + run: | + ls ${{ gitea.workspace }} + - run: echo "🍏 This job's status is ${{ job.status }}." +``` + +You can upload it as a file with the extension `.yaml` in the directory `.gitea/workflows/` of the repository, for example `.gitea/workflows/demo.yaml`. +You might notice that this is fairly similar from the [Quickstart for GitHub Actions](https://docs.github.com/en/actions/quickstart). +That is because Gitea Actions is designed to be compatible with GitHub Actions wherever possible. + +Be careful, the demo file contains some emojis. +Please make sure your database supports them, especially when using MySQL. +If the charset is not `utf8mb4`, errors will occur, such as `Error 1366 (HY000): Incorrect string value: '\\xF0\\x9F\\x8E\\x89 T...' for column 'name' at row 1`. +See [Database Preparation]({{< relref "doc/installation/database-preparation.en-us.md#mysql" >}}) for more information. + +Alternatively, you can remove all emojis from the demo file and try again. + +The line `on: [push]` indicates that the workflow will be triggered when you push commits to this repository. +However, when you upload the YAML file, it also pushes a commit, so you should see a new task in the Actions tab. + +![view job](/images/usage/actions/view-job.png) + +Great job! You have successfully started working with Actions. diff --git a/docs/content/doc/administration/command-line.en-us.md b/docs/content/doc/administration/command-line.en-us.md index bf4578afec5..37ba0c04da6 100644 --- a/docs/content/doc/administration/command-line.en-us.md +++ b/docs/content/doc/administration/command-line.en-us.md @@ -225,7 +225,7 @@ Admin operations: - `--synchronize-users`: Enable user synchronization. - `--page-size value`: Search page size. - Examples: - - `gitea admin auth add-ldap --name ldap --security-protocol unencrypted --host mydomain.org --port 389 --user-search-base "ou=Users,dc=mydomain,dc=org" --user-filter "(&(objectClass=posixAccount)(uid=%s))" --email-attribute mail` + - `gitea admin auth add-ldap --name ldap --security-protocol unencrypted --host mydomain.org --port 389 --user-search-base "ou=Users,dc=mydomain,dc=org" --user-filter "(&(objectClass=posixAccount)(|(uid=%[1]s)(mail=%[1]s)))" --email-attribute mail` - `update-ldap`: Update existing LDAP (via Bind DN) authentication source - Options: - `--id value`: ID of authentication source. Required. diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index b1e393b6c53..c1befed4898 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -89,10 +89,6 @@ In addition there is _`StaticRootPath`_ which can be set as a built-in at build - `DEFAULT_PUSH_CREATE_PRIVATE`: **true**: Default private when creating a new repository with push-to-create. - `MAX_CREATION_LIMIT`: **-1**: Global maximum creation limit of repositories per user, `-1` means no limit. -- `PULL_REQUEST_QUEUE_LENGTH`: **1000**: Length of pull request patch test queue, make it. **DEPRECATED** use `LENGTH` in `[queue.pr_patch_checker]`. - as large as possible. Use caution when editing this value. -- `MIRROR_QUEUE_LENGTH`: **1000**: Patch test queue length, increase if pull request patch - testing starts hanging. **DEPRECATED** use `LENGTH` in `[queue.mirror]`. - `PREFERRED_LICENSES`: **Apache License 2.0,MIT License**: Preferred Licenses to place at the top of the list. Name must match file name in options/license or custom/options/license. - `DISABLE_HTTP_GIT`: **false**: Disable the ability to interact with repositories over the @@ -368,7 +364,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a - `LFS_START_SERVER`: **false**: Enables Git LFS support. - `LFS_CONTENT_PATH`: **%(APP_DATA_PATH)s/lfs**: Default LFS content path. (if it is on local storage.) **DEPRECATED** use settings in `[lfs]`. - `LFS_JWT_SECRET`: **\**: LFS authentication secret, change this a unique string. -- `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail. +- `LFS_HTTP_AUTH_EXPIRY`: **24h**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail. - `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit). - `LFS_LOCKS_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page. @@ -465,11 +461,6 @@ relation to port exhaustion. - `ISSUE_INDEXER_CONN_STR`: ****: Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch, or meilisearch. i.e. http://elastic:changeme@localhost:9200 - `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_. -- The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility: -- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. **DEPRECATED** use settings in `[queue.issue_indexer]`. -- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. **DEPRECATED** use settings in `[queue.issue_indexer]`. Relative paths will be made absolute against `%(APP_DATA_PATH)s`. -- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. **DEPRECATED** use settings in `[queue.issue_indexer]`. -- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. @@ -480,7 +471,6 @@ relation to port exhaustion. - `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files. - `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`. - `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index. -- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. **DEPRECATED** use settings in `[queue.issue_indexer]`. - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed. - `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to -1 to never timeout. @@ -488,23 +478,14 @@ relation to port exhaustion. Configuration at `[queue]` will set defaults for queues with overrides for individual queues at `[queue.*]`. (However see below.) -- `TYPE`: **persistable-channel**: General queue type, currently support: `persistable-channel` (uses a LevelDB internally), `channel`, `level`, `redis`, `dummy` -- `DATADIR`: **queues/**: Base DataDir for storing persistent and level queues. `DATADIR` for individual queues can be set in `queue.name` sections but will default to `DATADIR/`**`common`**. (Previously each queue would default to `DATADIR/`**`name`**.) Relative paths will be made absolute against `%(APP_DATA_PATH)s`. -- `LENGTH`: **20**: Maximal queue size before channel queues block +- `TYPE`: **level**: General queue type, currently support: `level` (uses a LevelDB internally), `channel`, `redis`, `dummy`. Invalid types are treated as `level`. +- `DATADIR`: **queues/common**: Base DataDir for storing level queues. `DATADIR` for individual queues can be set in `queue.name` sections. Relative paths will be made absolute against `%(APP_DATA_PATH)s`. +- `LENGTH`: **100**: Maximal queue size before channel queues block - `BATCH_LENGTH`: **20**: Batch data before passing to the handler -- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. Options can be set using query params. Similarly LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR` +- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. Options can be set using query params. Similarly, LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR` - `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section. -- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to - **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section. -- `WRAP_IF_NECESSARY`: **true**: Will wrap queues with a timeoutable queue if the selected queue is not ready to be created - (Only relevant for the level queue.) -- `MAX_ATTEMPTS`: **10**: Maximum number of attempts to create the wrapped queue -- `TIMEOUT`: **GRACEFUL_HAMMER_TIME + 30s**: Timeout the creation of the wrapped queue if it takes longer than this to create. -- Queues by default come with a dynamically scaling worker pool. The following settings configure this: -- `WORKERS`: **0**: Number of initial workers for the queue. +- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section. - `MAX_WORKERS`: **10**: Maximum number of worker go-routines for the queue. -- `BLOCK_TIMEOUT`: **1s**: If the queue blocks for this time, boost the number of workers - the `BLOCK_TIMEOUT` will then be doubled before boosting again whilst the boost is ongoing. -- `BOOST_TIMEOUT`: **5m**: Boost workers will timeout after this long. -- `BOOST_WORKERS`: **1**: This many workers will be added to the worker pool if there is a boost. Gitea creates the following non-unique queues: @@ -522,21 +503,6 @@ And the following unique queues: - `mirror` - `pr_patch_checker` -Certain queues have defaults that override the defaults set in `[queue]` (this occurs mostly to support older configuration): - -- `[queue.issue_indexer]` - - `TYPE` this will default to `[queue]` `TYPE` if it is set but if not it will appropriately convert `[indexer]` `ISSUE_INDEXER_QUEUE_TYPE` if that is set. - - `LENGTH` will default to `[indexer]` `UPDATE_BUFFER_LEN` if that is set. - - `BATCH_LENGTH` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_BATCH_NUMBER` if that is set. - - `DATADIR` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_DIR` if that is set. - - `CONN_STR` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_CONN_STR` if that is set. -- `[queue.mailer]` - - `LENGTH` will default to **100** or whatever `[mailer]` `SEND_BUFFER_LEN` is. -- `[queue.pr_patch_checker]` - - `LENGTH` will default to **1000** or whatever `[repository]` `PULL_REQUEST_QUEUE_LENGTH` is. -- `[queue.mirror]` - - `LENGTH` will default to **1000** or whatever `[repository]` `MIRROR_QUEUE_LENGTH` is. - ## Admin (`admin`) - `DEFAULT_EMAIL_NOTIFICATIONS`: **enabled**: Default configuration for email notifications for users (user configurable). Options: enabled, onmention, disabled @@ -1259,6 +1225,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf - `LIMIT_SIZE_NUGET`: **-1**: Maximum size of a NuGet upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_PUB`: **-1**: Maximum size of a Pub upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_PYPI`: **-1**: Maximum size of a PyPI upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) +- `LIMIT_SIZE_RPM`: **-1**: Maximum size of a RPM upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_RUBYGEMS`: **-1**: Maximum size of a RubyGems upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_SWIFT`: **-1**: Maximum size of a Swift upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) - `LIMIT_SIZE_VAGRANT`: **-1**: Maximum size of a Vagrant upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`) diff --git a/docs/content/doc/administration/config-cheat-sheet.zh-cn.md b/docs/content/doc/administration/config-cheat-sheet.zh-cn.md index eb015908bbc..41eed612acc 100644 --- a/docs/content/doc/administration/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/administration/config-cheat-sheet.zh-cn.md @@ -43,7 +43,6 @@ menu: - `DEFAULT_PRIVATE`: 默认创建的git工程为私有。 可以是`last`, `private` 或 `public`。默认值是 `last`表示用户最后创建的Repo的选择。 - `DEFAULT_PUSH_CREATE_PRIVATE`: **true**: 通过 ``push-to-create`` 方式创建的仓库是否默认为私有仓库. - `MAX_CREATION_LIMIT`: 全局最大每个用户创建的git工程数目， `-1` 表示没限制。 -- `PULL_REQUEST_QUEUE_LENGTH`: 小心：合并请求测试队列的长度，尽量放大。 ### Repository - Release (`repository.release`) @@ -111,10 +110,6 @@ menu: - `ISSUE_INDEXER_CONN_STR`: ****: 工单索引连接字符串，仅当 ISSUE_INDEXER_TYPE 为 `elasticsearch` 时有效。例如: http://elastic:changeme@localhost:9200 - `ISSUE_INDEXER_NAME`: **gitea_issues**: 工单索引名称，仅当 ISSUE_INDEXER_TYPE 为 `elasticsearch` 时有效。 - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: 工单索引文件存放路径，当索引类型为 `bleve` 时有效。 -- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: 工单索引队列类型，当前支持 `channel`， `levelqueue` 或 `redis`。 -- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: 当 `ISSUE_INDEXER_QUEUE_TYPE` 为 `levelqueue` 时，保存索引队列的磁盘路径。 -- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 当 `ISSUE_INDEXER_QUEUE_TYPE` 为 `redis` 时，保存Redis队列的连接字符串。 -- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: 队列处理中批量提交数量。 - `REPO_INDEXER_ENABLED`: **false**: 是否启用代码搜索（启用后会占用比较大的磁盘空间，如果是bleve可能需要占用约6倍存储空间）。 - `REPO_INDEXER_TYPE`: **bleve**: 代码搜索引擎类型，可以为 `bleve` 或者 `elasticsearch`。 @@ -122,7 +117,6 @@ menu: - `REPO_INDEXER_CONN_STR`: ****: 代码搜索引擎连接字符串，当 `REPO_INDEXER_TYPE` 为 `elasticsearch` 时有效。例如： http://elastic:changeme@localhost:9200 - `REPO_INDEXER_NAME`: **gitea_codes**: 代码搜索引擎的名字，当 `REPO_INDEXER_TYPE` 为 `elasticsearch` 时有效。 -- `UPDATE_BUFFER_LEN`: **20**: 代码索引请求的缓冲区长度。 - `MAX_FILE_SIZE`: **1048576**: 进行解析的源代码文件的最大长度，小于该值时才会索引。 ## Security (`security`) diff --git a/docs/content/doc/administration/repo-indexer.en-us.md b/docs/content/doc/administration/repo-indexer.en-us.md index 81d22434767..a1980bc5fed 100644 --- a/docs/content/doc/administration/repo-indexer.en-us.md +++ b/docs/content/doc/administration/repo-indexer.en-us.md @@ -30,7 +30,6 @@ Gitea can search through the files of the repositories by enabling this function ; ... REPO_INDEXER_ENABLED = true REPO_INDEXER_PATH = indexers/repos.bleve -UPDATE_BUFFER_LEN = 20 MAX_FILE_SIZE = 1048576 REPO_INDEXER_INCLUDE = REPO_INDEXER_EXCLUDE = resources/bin/** diff --git a/docs/content/doc/administration/reverse-proxies.en-us.md b/docs/content/doc/administration/reverse-proxies.en-us.md index d11552bb9fa..a6efd86549a 100644 --- a/docs/content/doc/administration/reverse-proxies.en-us.md +++ b/docs/content/doc/administration/reverse-proxies.en-us.md @@ -25,12 +25,13 @@ menu: If you want Nginx to serve your Gitea instance, add the following `server` section to the `http` section of `nginx.conf`: -```apacheconf +``` server { listen 80; server_name git.example.com; location / { + client_max_body_size 512M; proxy_pass http://localhost:3000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -40,23 +41,32 @@ server { } ``` +### Resolving Error: 413 Request Entity Too Large + +This error indicates nginx is configured to restrict the file upload size, +it affects attachment uploading, form posting, package uploading and LFS pushing, etc. +You can fine tune the `client_max_body_size` option according to [nginx document](http://nginx.org/en/docs/http/ngx_http_core_module.html#client_max_body_size). + ## Nginx with a sub-path In case you already have a site, and you want Gitea to share the domain name, you can setup Nginx to serve Gitea under a sub-path by adding the following `server` section inside the `http` section of `nginx.conf`: -```apacheconf +``` server { listen 80; server_name git.example.com; # Note: Trailing slash - location /git/ { - # Note: Trailing slash - proxy_pass http://localhost:3000/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + location /gitea/ { + client_max_body_size 512M; + + # make nginx use unescaped URI, keep "%2F" as is + rewrite ^ $request_uri; + rewrite ^/gitea(/.*) $1 break; + proxy_pass http://127.0.0.1:3000$uri; + + # other common HTTP headers, see the "Nginx" config section above + proxy_set_header ... } } ``` @@ -132,14 +142,6 @@ server { } ``` -## Resolving Error: 413 Request Entity Too Large - -This error indicates nginx is configured to restrict the file upload size. - -In your nginx config file containing your Gitea proxy directive, find the `location { ... }` block for Gitea and add the line -`client_max_body_size 16M;` to set this limit to 16 megabytes or any other number of choice. -If you use Git LFS, this will also limit the size of the largest file you will be able to push. - ## Apache HTTPD If you want Apache HTTPD to serve your Gitea instance, you can add the following to your Apache HTTPD configuration (usually located at `/etc/apache2/httpd.conf` in Ubuntu): @@ -387,3 +389,13 @@ gitea: This config assumes that you are handling HTTPS on the traefik side and using HTTP between Gitea and traefik. Then you **MUST** set something like `[server] ROOT_URL = http://example.com/gitea/` correctly in your configuration. + +## General sub-path configuration + +Usually it's not recommended to put Gitea in a sub-path, it's not widely used and may have some issues in rare cases. + +If you really need to do so, to make Gitea works with sub-path (eg: `http://example.com/gitea/`), here are the requirements: + +1. Set `[server] ROOT_URL = http://example.com/gitea/` in your `app.ini` file. +2. Make the reverse-proxy pass `http://example.com/gitea/foo` to `http://gitea-server:3000/foo`. +3. Make sure the reverse-proxy not decode the URI, the request `http://example.com/gitea/a%2Fb` should be passed as `http://gitea-server:3000/a%2Fb`. diff --git a/docs/content/doc/development/api-usage.en-us.md b/docs/content/doc/development/api-usage.en-us.md index fe334827c3f..4f5304ac0e9 100644 --- a/docs/content/doc/development/api-usage.en-us.md +++ b/docs/content/doc/development/api-usage.en-us.md @@ -119,7 +119,7 @@ curl -v "http://localhost/api/v1/repos/search?limit=1" < x-total-count: 5252 ``` -## API Guide: +## API Guide API Reference guide is auto-generated by swagger and available on: `https://gitea.your.host/api/swagger` diff --git a/docs/content/doc/development/hacking-on-gitea.zh-cn.md b/docs/content/doc/development/hacking-on-gitea.zh-cn.md index 3a4078dfdb6..6a18777ed68 100644 --- a/docs/content/doc/development/hacking-on-gitea.zh-cn.md +++ b/docs/content/doc/development/hacking-on-gitea.zh-cn.md @@ -168,7 +168,7 @@ make lint-backend ### 处理 JS 和 CSS -前端开发应遵循 [Guidelines for Frontend Development]({{ < 相关参考 "doc/development/guidelines-frontend.en-us.md" > }}) +前端开发应遵循 [Guidelines for Frontend Development]({{< relref "doc/contributing/guidelines-frontend.zh-cn.md" >}})。 要使用前端资源构建，请使用上面提到的“watch-frontend”目标或只构建一次： diff --git a/docs/content/doc/usage/authentication.en-us.md b/docs/content/doc/usage/authentication.en-us.md index 2b8cdd29642..d9648200ef7 100644 --- a/docs/content/doc/usage/authentication.en-us.md +++ b/docs/content/doc/usage/authentication.en-us.md @@ -100,9 +100,9 @@ Adds the following fields: - User Filter **(required)** - An LDAP filter declaring how to find the user record that is attempting to - authenticate. The `%s` matching parameter will be substituted with login + authenticate. The `%[1]s` matching parameter will be substituted with login name given on sign-in form. - - Example: `(&(objectClass=posixAccount)(uid=%s))` + - Example: `(&(objectClass=posixAccount)(|(uid=%[1]s)(mail=%[1]s)))` - Example for Microsoft Active Directory (AD): `(&(objectCategory=Person)(memberOf=CN=user-group,OU=example,DC=example,DC=org)(sAMAccountName=%s)(!(UserAccountControl:1.2.840.113556.1.4.803:=2)))` - To substitute more than once, `%[1]s` should be used instead, e.g. when matching supplied login name against multiple attributes such as user @@ -137,11 +137,11 @@ Adds the following fields: - Example: `ou=Users,dc=mydomain,dc=com` - User Filter **(required)** - - An LDAP filter declaring when a user should be allowed to log in. The `%s` + - An LDAP filter declaring when a user should be allowed to log in. The `%[1]s` matching parameter will be substituted with login name given on sign-in form. - - Example: `(&(objectClass=posixAccount)(cn=%s))` - - Example: `(&(objectClass=posixAccount)(uid=%s))` + - Example: `(&(objectClass=posixAccount)(|(cn=%[1]s)(mail=%[1]s)))` + - Example: `(&(objectClass=posixAccount)(|(uid=%[1]s)(mail=%[1]s)))` ### Verify group membership in LDAP diff --git a/docs/content/doc/usage/packages/overview.en-us.md b/docs/content/doc/usage/packages/overview.en-us.md index 8e4fd87e7b7..8a70a352eb3 100644 --- a/docs/content/doc/usage/packages/overview.en-us.md +++ b/docs/content/doc/usage/packages/overview.en-us.md @@ -41,6 +41,7 @@ The following package managers are currently supported: | [NuGet]({{< relref "doc/usage/packages/nuget.en-us.md" >}}) | .NET | `nuget` | | [Pub]({{< relref "doc/usage/packages/pub.en-us.md" >}}) | Dart | `dart`, `flutter` | | [PyPI]({{< relref "doc/usage/packages/pypi.en-us.md" >}}) | Python | `pip`, `twine` | +| [RPM]({{< relref "doc/usage/packages/rpm.en-us.md" >}}) | - | `yum`, `dnf` | | [RubyGems]({{< relref "doc/usage/packages/rubygems.en-us.md" >}}) | Ruby | `gem`, `Bundler` | | [Swift]({{< relref "doc/usage/packages/rubygems.en-us.md" >}}) | Swift | `swift` | | [Vagrant]({{< relref "doc/usage/packages/vagrant.en-us.md" >}}) | - | `vagrant` | diff --git a/docs/content/doc/usage/packages/rpm.en-us.md b/docs/content/doc/usage/packages/rpm.en-us.md new file mode 100644 index 00000000000..2f9bb539bef --- /dev/null +++ b/docs/content/doc/usage/packages/rpm.en-us.md @@ -0,0 +1,118 @@ +--- +date: "2023-03-08T00:00:00+00:00" +title: "RPM Packages Repository" +slug: "packages/rpm" +draft: false +toc: false +menu: + sidebar: + parent: "packages" + name: "RPM" + weight: 105 + identifier: "rpm" +--- + +# RPM Packages Repository + +Publish [RPM](https://rpm.org/) packages for your user or organization. + +**Table of Contents** + +{{< toc >}} + +## Requirements + +To work with the RPM registry, you need to use a package manager like `yum` or `dnf` to consume packages. + +The following examples use `dnf`. + +## Configuring the package registry + +To register the RPM registry add the url to the list of known apt sources: + +```shell +dnf config-manager --add-repo https://gitea.example.com/api/packages/{owner}/rpm.repo +``` + +| Placeholder | Description | +| ----------- | ----------- | +| `owner` | The owner of the package. | + +If the registry is private, provide credentials in the url. You can use a password or a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}): + +```shell +dnf config-manager --add-repo https://{username}:{your_password_or_token}@gitea.example.com/api/packages/{owner}/rpm.repo +``` + +You have to add the credentials to the urls in the `rpm.repo` file in `/etc/yum.repos.d` too. + +## Publish a package + +To publish a RPM package (`*.rpm`), perform a HTTP PUT operation with the package content in the request body. + +``` +PUT https://gitea.example.com/api/packages/{owner}/rpm/upload +``` + +| Parameter | Description | +| --------- | ----------- | +| `owner` | The owner of the package. | + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_password_or_token \ + --upload-file path/to/file.rpm \ + https://gitea.example.com/api/packages/testuser/rpm/upload +``` + +If you are using 2FA or OAuth use a [personal access token]({{< relref "doc/development/api-usage.en-us.md#authentication" >}}) instead of the password. +You cannot publish a file with the same name twice to a package. You must delete the existing package version first. + +The server reponds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `201 Created` | The package has been published. | +| `400 Bad Request` | The package is invalid. | +| `409 Conflict` | A package file with the same combination of parameters exist already in the package. | + +## Delete a package + +To delete a Debian package perform a HTTP DELETE operation. This will delete the package version too if there is no file left. + +``` +DELETE https://gitea.example.com/api/packages/{owner}/rpm/{package_name}/{package_version}/{architecture} +``` + +| Parameter | Description | +| ----------------- | ----------- | +| `owner` | The owner of the package. | +| `package_name` | The package name. | +| `package_version` | The package version. | +| `architecture` | The package architecture. | + +Example request using HTTP Basic authentication: + +```shell +curl --user your_username:your_token_or_password -X DELETE \ + https://gitea.example.com/api/packages/testuser/rpm/test-package/1.0.0/x86_64 +``` + +The server reponds with the following HTTP Status codes. + +| HTTP Status Code | Meaning | +| ----------------- | ------- | +| `204 No Content` | Success | +| `404 Not Found` | The package or file was not found. | + +## Install a package + +To install a package from the RPM registry, execute the following commands: + +```shell +# use latest version +dnf install {package_name} +# use specific version +dnf install {package_name}-{package_version}.{architecture} +``` diff --git a/docs/content/doc/usage/packages/swift.en-us.md b/docs/content/doc/usage/packages/swift.en-us.md index a1c818021cd..3164f8d1701 100644 --- a/docs/content/doc/usage/packages/swift.en-us.md +++ b/docs/content/doc/usage/packages/swift.en-us.md @@ -15,7 +15,7 @@ menu: # Swift Packages Repository -Publish [Swift](hhttps://www.swift.org/) packages for your user or organization. +Publish [Swift](https://www.swift.org/) packages for your user or organization. **Table of Contents** diff --git a/docs/content/doc/usage/profile-readme.en-us.md b/docs/content/doc/usage/profile-readme.en-us.md new file mode 100644 index 00000000000..802451f0b69 --- /dev/null +++ b/docs/content/doc/usage/profile-readme.en-us.md @@ -0,0 +1,20 @@ +--- +date: "2023-03-02T21:00:00+05:00" +title: "Usage: Gitea Profile READMEs" +slug: "profile-readme" +weight: 12 +toc: false +draft: false +menu: + sidebar: + parent: "usage" + name: "Gitea Profile READMEs" + weight: 12 + identifier: "profile-readme" +--- + +# Gitea Profile READMEs + +To display a markdown file in your Gitea profile page, simply make a repository named ".profile" and edit the README.md file inside. Gitea will automatically pull this file in and display it above your repositories. + +Note. You are welcome to make this repository private. Doing so will hide your source files from public viewing and allow you to privitize certain files. However, the README.md file will be the only file present on your profile. If you wish to have an entirely private .profile repository, remove or rename the README.md file. diff --git a/docs/static/images/usage/actions/enable-actions.png b/docs/static/images/usage/actions/enable-actions.png new file mode 100644 index 00000000000..8e38bd9a8df Binary files /dev/null and b/docs/static/images/usage/actions/enable-actions.png differ diff --git a/docs/static/images/usage/actions/labels.png b/docs/static/images/usage/actions/labels.png new file mode 100644 index 00000000000..d6e847950b9 Binary files /dev/null and b/docs/static/images/usage/actions/labels.png differ diff --git a/docs/static/images/usage/actions/network.png b/docs/static/images/usage/actions/network.png new file mode 100644 index 00000000000..afaa5ddd3d6 Binary files /dev/null and b/docs/static/images/usage/actions/network.png differ diff --git a/docs/static/images/usage/actions/register-runner.png b/docs/static/images/usage/actions/register-runner.png new file mode 100644 index 00000000000..a260e44f483 Binary files /dev/null and b/docs/static/images/usage/actions/register-runner.png differ diff --git a/docs/static/images/usage/actions/view-job.png b/docs/static/images/usage/actions/view-job.png new file mode 100644 index 00000000000..493869969be Binary files /dev/null and b/docs/static/images/usage/actions/view-job.png differ diff --git a/docs/static/images/usage/actions/view-runner.png b/docs/static/images/usage/actions/view-runner.png new file mode 100644 index 00000000000..f25d2643f73 Binary files /dev/null and b/docs/static/images/usage/actions/view-runner.png differ diff --git a/go.mod b/go.mod index 58e90cc0fd3..24765df1813 100644 --- a/go.mod +++ b/go.mod @@ -7,20 +7,20 @@ require ( code.gitea.io/gitea-vet v0.2.2 code.gitea.io/sdk/gitea v0.15.1 codeberg.org/gusted/mcaptcha v0.0.0-20220723083913-4f3072e1d570 - gitea.com/go-chi/binding v0.0.0-20221013104517-b29891619681 + gitea.com/go-chi/binding v0.0.0-20230415142243-04b515c6d669 gitea.com/go-chi/cache v0.2.0 - gitea.com/go-chi/captcha v0.0.0-20211013065431-70641c1a35d5 - gitea.com/go-chi/session v0.0.0-20221220005550-e056dc379164 + gitea.com/go-chi/captcha v0.0.0-20230415143339-2c0754df4384 + gitea.com/go-chi/session v0.0.0-20230415140235-3182bcc14852 gitea.com/lunny/dingtalk_webhook v0.0.0-20171025031554-e3534c89ef96 gitea.com/lunny/levelqueue v0.4.2-0.20220729054728-f020868cc2f7 github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 github.com/NYTimes/gziphandler v1.1.1 - github.com/PuerkitoBio/goquery v1.8.0 - github.com/alecthomas/chroma/v2 v2.5.0 + github.com/PuerkitoBio/goquery v1.8.1 + github.com/alecthomas/chroma/v2 v2.7.0 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb - github.com/blevesearch/bleve/v2 v2.3.6 - github.com/bufbuild/connect-go v1.3.1 + github.com/blevesearch/bleve/v2 v2.3.7 + github.com/bufbuild/connect-go v1.7.0 github.com/buildkite/terminal-to-html/v3 v3.7.0 github.com/caddyserver/certmagic v0.17.2 github.com/chi-middleware/proxy v1.1.1 @@ -30,91 +30,92 @@ require ( github.com/djherbis/nio/v3 v3.0.1 github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 github.com/dustin/go-humanize v1.0.1 - github.com/editorconfig/editorconfig-core-go/v2 v2.5.1 + github.com/editorconfig/editorconfig-core-go/v2 v2.5.2 github.com/emersion/go-imap v1.2.1 github.com/emirpasic/gods v1.18.1 github.com/ethantkoenig/rupture v1.0.1 github.com/felixge/fgprof v0.9.3 github.com/fsnotify/fsnotify v1.6.0 github.com/gliderlabs/ssh v0.3.5 - github.com/go-ap/activitypub v0.0.0-20230218112952-bfb607b04799 + github.com/go-ap/activitypub v0.0.0-20230331173947-f5b96d9450d4 github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73 github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/cors v1.2.1 - github.com/go-enry/go-enry/v2 v2.8.3 + github.com/go-enry/go-enry/v2 v2.8.4 github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e github.com/go-git/go-billy/v5 v5.4.1 - github.com/go-git/go-git/v5 v5.5.2 + github.com/go-git/go-git/v5 v5.6.1 github.com/go-ldap/ldap/v3 v3.4.4 - github.com/go-sql-driver/mysql v1.7.0 + github.com/go-sql-driver/mysql v1.7.1 github.com/go-swagger/go-swagger v0.30.4 - github.com/go-testfixtures/testfixtures/v3 v3.8.1 - github.com/go-webauthn/webauthn v0.8.1 + github.com/go-testfixtures/testfixtures/v3 v3.9.0 + github.com/go-webauthn/webauthn v0.8.2 github.com/gobwas/glob v0.2.3 github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/go-github/v51 v51.0.0 - github.com/google/pprof v0.0.0-20230222194610-99052d3372e7 + github.com/google/pprof v0.0.0-20230502171905-255e3b9b56de github.com/google/uuid v1.3.0 github.com/gorilla/feeds v1.1.1 github.com/gorilla/sessions v1.2.1 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/golang-lru v0.6.0 github.com/huandu/xstrings v1.4.0 - github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba - github.com/jhillyerd/enmime v0.10.1 + github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 + github.com/jhillyerd/enmime v0.11.1 github.com/json-iterator/go v1.1.12 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/keybase/go-crypto v0.0.0-20200123153347-de78d2cb44f4 - github.com/klauspost/compress v1.15.15 + github.com/klauspost/compress v1.16.5 github.com/klauspost/cpuid/v2 v2.2.4 - github.com/lib/pq v1.10.7 - github.com/markbates/goth v1.76.0 - github.com/mattn/go-isatty v0.0.17 + github.com/lib/pq v1.10.9 + github.com/markbates/goth v1.77.0 + github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.16 - github.com/meilisearch/meilisearch-go v0.23.0 + github.com/meilisearch/meilisearch-go v0.24.0 github.com/mholt/archiver/v3 v3.5.1 - github.com/microcosm-cc/bluemonday v1.0.22 - github.com/minio/minio-go/v7 v7.0.49 + github.com/microcosm-cc/bluemonday v1.0.23 + github.com/minio/minio-go/v7 v7.0.52 github.com/minio/sha256-simd v1.0.0 github.com/msteinert/pam v1.1.0 - github.com/nektos/act v0.2.43 + github.com/nektos/act v0.2.45 github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 github.com/niklasfasching/go-org v1.6.5 github.com/oliamb/cutter v0.2.2 github.com/olivere/elastic/v7 v7.0.32 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc2 + github.com/opencontainers/image-spec v1.1.0-rc3 github.com/pkg/errors v0.9.1 github.com/pquerna/otp v1.4.0 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.15.1 github.com/quasoft/websspi v1.1.2 - github.com/redis/go-redis/v9 v9.0.3 - github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 + github.com/redis/go-redis/v9 v9.0.4 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 + github.com/sassoftware/go-rpmutils v0.2.0 github.com/sergi/go-diff v1.3.1 github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 github.com/ulikunitz/xz v0.5.11 - github.com/urfave/cli v1.22.12 - github.com/xanzy/go-gitlab v0.80.2 + github.com/urfave/cli v1.22.13 + github.com/xanzy/go-gitlab v0.83.0 github.com/xeipuuv/gojsonschema v1.2.0 - github.com/yohcop/openid-go v1.0.0 + github.com/yohcop/openid-go v1.0.1 github.com/yuin/goldmark v1.5.4 github.com/yuin/goldmark-highlighting/v2 v2.0.0-20220924101305-151362477c87 github.com/yuin/goldmark-meta v1.1.0 - golang.org/x/crypto v0.7.0 + golang.org/x/crypto v0.8.0 golang.org/x/image v0.7.0 - golang.org/x/net v0.8.0 - golang.org/x/oauth2 v0.6.0 - golang.org/x/sys v0.6.0 + golang.org/x/net v0.9.0 + golang.org/x/oauth2 v0.7.0 + golang.org/x/sys v0.8.0 golang.org/x/text v0.9.0 - golang.org/x/tools v0.6.0 + golang.org/x/tools v0.8.0 google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.28.1 + google.golang.org/protobuf v1.30.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v3 v3.0.1 @@ -128,20 +129,23 @@ require ( cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 // indirect + github.com/ClickHouse/ch-go v0.55.0 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.9.1 // indirect + github.com/DataDog/zstd v1.4.5 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Masterminds/sprig/v3 v3.2.3 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect - github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230426101702-58e86b294756 // indirect github.com/RoaringBitmap/roaring v1.2.3 // indirect - github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/acomagu/bufpipe v1.0.4 // indirect github.com/andybalholm/brotli v1.0.5 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.5.0 // indirect + github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/blevesearch/bleve_index_api v1.0.5 // indirect github.com/blevesearch/geo v0.1.17 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect @@ -156,26 +160,28 @@ require ( github.com/blevesearch/zapx/v12 v12.3.7 // indirect github.com/blevesearch/zapx/v13 v13.3.7 // indirect github.com/blevesearch/zapx/v14 v14.3.7 // indirect - github.com/blevesearch/zapx/v15 v15.3.9 // indirect + github.com/blevesearch/zapx/v15 v15.3.10 // indirect github.com/boombuler/barcode v1.0.1 // indirect github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cloudflare/circl v1.3.2 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/couchbase/go-couchbase v0.0.0-20210224140812-5740cd35f448 // indirect github.com/couchbase/gomemcached v0.1.2 // indirect github.com/couchbase/goutils v0.0.0-20210118111533-e33d3ffb5401 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dlclark/regexp2 v1.8.1 // indirect + github.com/dlclark/regexp2 v1.9.0 // indirect github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21 // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/go-ap/errors v0.0.0-20221205040414-01c1adfc98ea // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/go-enry/go-oniguruma v1.2.1 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.6.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-openapi/analysis v0.21.4 // indirect github.com/go-openapi/errors v0.20.3 // indirect @@ -189,11 +195,11 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/go-openapi/validate v0.22.0 // indirect github.com/go-webauthn/revoke v0.1.9 // indirect - github.com/goccy/go-json v0.10.0 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect - github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-tpm v0.3.3 // indirect @@ -206,12 +212,12 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.13 // indirect + github.com/imdario/mergo v0.3.15 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jessevdk/go-flags v1.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/libdns/libdns v0.2.1 // indirect @@ -220,9 +226,9 @@ require ( github.com/markbates/going v1.0.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/mholt/acmez v1.0.4 // indirect - github.com/miekg/dns v1.1.50 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mholt/acmez v1.1.0 // indirect + github.com/miekg/dns v1.1.54 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -236,21 +242,23 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.18.1 // indirect + github.com/paulmach/orb v0.9.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect - github.com/pjbgf/sha1cd v0.2.3 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/rhysd/actionlint v1.6.23 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + github.com/rhysd/actionlint v1.6.24 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/robfig/cron v1.2.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect - github.com/rs/xid v1.4.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/shopspring/decimal v1.2.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.3.1 // indirect github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/skeema/knownhosts v1.1.0 // indirect @@ -264,7 +272,7 @@ require ( github.com/toqueteos/webbrowser v1.2.0 // indirect github.com/unknwon/com v1.0.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.44.0 // indirect + github.com/valyala/fasthttp v1.47.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -272,15 +280,18 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect go.etcd.io/bbolt v1.3.7 // indirect - go.mongodb.org/mongo-driver v1.11.1 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.mongodb.org/mongo-driver v1.11.4 // indirect + go.opentelemetry.io/otel v1.15.1 // indirect + go.opentelemetry.io/otel/trace v1.15.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/mod v0.8.0 // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 // indirect + google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -290,8 +301,6 @@ replace github.com/hashicorp/go-version => github.com/6543/go-version v1.3.1 replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142115-2c99e1ffdfa0 -replace github.com/blevesearch/zapx/v15 v15.3.6 => github.com/zeripath/zapx/v15 v15.3.6-alignment-fix - replace github.com/nektos/act => gitea.com/gitea/act v0.243.4 exclude github.com/gofrs/uuid v3.2.0+incompatible diff --git a/go.sum b/go.sum index ba969dddeef..62c01c690af 100644 --- a/go.sum +++ b/go.sum @@ -54,15 +54,15 @@ git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 h1:cliQ4H git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078/go.mod h1:g/V2Hjas6Z1UHUp4yIx6bATpNzJ7DYtD0FG3+xARWxs= gitea.com/gitea/act v0.243.4 h1:MuBHBLCJfpa6mzwwvs4xqQynrSP2RRzpHpWfTV16PmI= gitea.com/gitea/act v0.243.4/go.mod h1:mabw6AZAiDgxGlK83orWLrNERSPvgBJzEUS3S7u2bHI= -gitea.com/go-chi/binding v0.0.0-20221013104517-b29891619681 h1:MMSPgnVULVwV9kEBgvyEUhC9v/uviZ55hPJEMjpbNR4= -gitea.com/go-chi/binding v0.0.0-20221013104517-b29891619681/go.mod h1:77TZu701zMXWJFvB8gvTbQ92zQ3DQq/H7l5wAEjQRKc= +gitea.com/go-chi/binding v0.0.0-20230415142243-04b515c6d669 h1:RUBX+MK/TsDxpHmymaOaydfigEbbzqUnG1OTZU/HAeo= +gitea.com/go-chi/binding v0.0.0-20230415142243-04b515c6d669/go.mod h1:77TZu701zMXWJFvB8gvTbQ92zQ3DQq/H7l5wAEjQRKc= gitea.com/go-chi/cache v0.0.0-20210110083709-82c4c9ce2d5e/go.mod h1:k2V/gPDEtXGjjMGuBJiapffAXTv76H4snSmlJRLUhH0= gitea.com/go-chi/cache v0.2.0 h1:E0npuTfDW6CT1yD8NMDVc1SK6IeRjfmRL2zlEsCEd7w= gitea.com/go-chi/cache v0.2.0/go.mod h1:iQlVK2aKTZ/rE9UcHyz9pQWGvdP9i1eI2spOpzgCrtE= -gitea.com/go-chi/captcha v0.0.0-20211013065431-70641c1a35d5 h1:J/1i8u40TbcLP/w2w2KCkgW2PelIqYkD5UOwu8IOvVU= -gitea.com/go-chi/captcha v0.0.0-20211013065431-70641c1a35d5/go.mod h1:hQ9SYHKdOX968wJglb/NMQ+UqpOKwW4L+EYdvkWjHSo= -gitea.com/go-chi/session v0.0.0-20221220005550-e056dc379164 h1:nvIOUfSzrQ6EphSLnYI+e4OcvqTd6HWdCnh+aIq9Uew= -gitea.com/go-chi/session v0.0.0-20221220005550-e056dc379164/go.mod h1:fc/pjt5EqNKgqQXYzcas1Z5L5whkZHyOvTA7OzWVJck= +gitea.com/go-chi/captcha v0.0.0-20230415143339-2c0754df4384 h1:klh0LjhH7l4CuJkxlCM//o3rWLvWqxUpFxEtoYg5TNY= +gitea.com/go-chi/captcha v0.0.0-20230415143339-2c0754df4384/go.mod h1:hQ9SYHKdOX968wJglb/NMQ+UqpOKwW4L+EYdvkWjHSo= +gitea.com/go-chi/session v0.0.0-20230415140235-3182bcc14852 h1:K/QcpdrEbOXZgYhcUrd0+S1j0MsZ1/rSV8Cy2+DdsK8= +gitea.com/go-chi/session v0.0.0-20230415140235-3182bcc14852/go.mod h1:fc/pjt5EqNKgqQXYzcas1Z5L5whkZHyOvTA7OzWVJck= gitea.com/lunny/dingtalk_webhook v0.0.0-20171025031554-e3534c89ef96 h1:+wWBi6Qfruqu7xJgjOIrKVQGiLUZdpKYCZewJ4clqhw= gitea.com/lunny/dingtalk_webhook v0.0.0-20171025031554-e3534c89ef96/go.mod h1:VyMQP6ue6MKHM8UsOXfNfuMKD0oSAWZdXVcpHIN2yaY= gitea.com/lunny/levelqueue v0.4.2-0.20220729054728-f020868cc2f7 h1:Zc3RQWC2xOVglLciQH/ZIC5IqSk3Jn96LflGQLv18Rg= @@ -83,6 +83,12 @@ github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzS github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/ClickHouse/ch-go v0.55.0 h1:jw4Tpx887YXrkyL5DfgUome/po8MLz92nz2heOQ6RjQ= +github.com/ClickHouse/ch-go v0.55.0/go.mod h1:kQT2f+yp2p+sagQA/7kS6G3ukym+GQ5KAu1kuFAFDiU= +github.com/ClickHouse/clickhouse-go/v2 v2.9.1 h1:IeE2bwVvAba7Yw5ZKu98bKI4NpDmykEy6jUaQdJJCk8= +github.com/ClickHouse/clickhouse-go/v2 v2.9.1/go.mod h1:teXfZNM90iQ99Jnuht+dxQXCuhDZ8nvvMoTJOFrcmcg= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -93,16 +99,16 @@ github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYr github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= -github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= -github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= -github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/ProtonMail/go-crypto v0.0.0-20230426101702-58e86b294756 h1:L6S7kR7SlhQKplIBpkra3s6yhcZV51lhRnXmYc4HohI= +github.com/ProtonMail/go-crypto v0.0.0-20230426101702-58e86b294756/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= @@ -112,26 +118,26 @@ github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= -github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= +github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/alecthomas/assert/v2 v2.2.1 h1:XivOgYcduV98QCahG8T5XTezV5bylXe+lBxLG2K2ink= github.com/alecthomas/chroma/v2 v2.2.0/go.mod h1:vf4zrexSH54oEjJ7EdB65tGNHmH3pGZmVkgTP5RHvAs= -github.com/alecthomas/chroma/v2 v2.5.0 h1:CQCdj1BiBV17sD4Bd32b/Bzuiq/EqoNTrnIhyQAZ+Rk= -github.com/alecthomas/chroma/v2 v2.5.0/go.mod h1:yrkMI9807G1ROx13fhe1v6PN2DDeaR73L3d+1nmYQtw= +github.com/alecthomas/chroma/v2 v2.7.0 h1:hm1rY6c/Ob4eGclpQ7X/A3yhqBOZNUTk9q+yhyLIViI= +github.com/alecthomas/chroma/v2 v2.7.0/go.mod h1:yrkMI9807G1ROx13fhe1v6PN2DDeaR73L3d+1nmYQtw= github.com/alecthomas/repr v0.0.0-20220113201626-b1b626ac65ae/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -152,7 +158,6 @@ github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZw github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -160,13 +165,13 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.1.10/go.mod h1:w0XsmFg8qg6cmpTtJ0z3pKgjTDBMMnI/+I2syrE6XBE= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= -github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= +github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/blevesearch/bleve/v2 v2.0.5/go.mod h1:ZjWibgnbRX33c+vBRgla9QhPb4QOjD6fdVJ+R1Bk8LM= -github.com/blevesearch/bleve/v2 v2.3.6 h1:NlntUHcV5CSWIhpugx4d/BRMGCiaoI8ZZXrXlahzNq4= -github.com/blevesearch/bleve/v2 v2.3.6/go.mod h1:JM2legf1cKVkdV8Ehu7msKIOKC0McSw0Q16Fmv9vsW4= +github.com/blevesearch/bleve/v2 v2.3.7 h1:nIfIrhv28tvgBpbVF8Dq7/U1zW/YiwSqg/PBgE3x8bo= +github.com/blevesearch/bleve/v2 v2.3.7/go.mod h1:2tToYD6mDeseIA13jcZiEEqYrVLg6xdk0v6+F7dWquU= github.com/blevesearch/bleve_index_api v1.0.0/go.mod h1:fiwKS0xLEm+gBRgv5mumf0dhgFr2mDgZah1pqv1c1M4= github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw= github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= @@ -207,8 +212,8 @@ github.com/blevesearch/zapx/v14 v14.2.0/go.mod h1:GNgZusc1p4ot040cBQMRGEZobvwjCq github.com/blevesearch/zapx/v14 v14.3.7 h1:gfe+fbWslDWP/evHLtp/GOvmNM3sw1BbqD7LhycBX20= github.com/blevesearch/zapx/v14 v14.3.7/go.mod h1:9J/RbOkqZ1KSjmkOes03AkETX7hrXT0sFMpWH4ewC4w= github.com/blevesearch/zapx/v15 v15.2.0/go.mod h1:MmQceLpWfME4n1WrBFIwplhWmaQbQqLQARpaKUEOs/A= -github.com/blevesearch/zapx/v15 v15.3.9 h1:/s9zqKxFaZKQTTcMO2b/Tup0ch5MSztlvw+frVDfIBk= -github.com/blevesearch/zapx/v15 v15.3.9/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY= +github.com/blevesearch/zapx/v15 v15.3.10 h1:bQ9ZxJCj6rKp873EuVJu2JPxQ+EWQZI1cjJGeroovaQ= +github.com/blevesearch/zapx/v15 v15.3.10/go.mod h1:m7Y6m8soYUvS7MjN9eKlz1xrLCcmqfFadmu7GhWIrLY= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1 h1:NDBbPmhS+EqABEs5Kg3n/5ZNjy73Pz7SIV+KCeqyXcs= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= @@ -217,8 +222,8 @@ github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b h1:L/QXpzIa3pO github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= -github.com/bufbuild/connect-go v1.3.1 h1:doJP6Q8Ypg6haUT2IAZJPWHUN9rAUp+F9MfK7yhu1zs= -github.com/bufbuild/connect-go v1.3.1/go.mod h1:9iNvh/NOsfhNBUH5CtvXeVUskQO1xsrEviH7ZArwZ3I= +github.com/bufbuild/connect-go v1.7.0 h1:MGp82v7SCza+3RhsVhV7aMikwxvI3ZfD72YiGt8FYJo= +github.com/bufbuild/connect-go v1.7.0/go.mod h1:GmMJYR6orFqD0Y6ZgX8pwQ8j9baizDrIQMm1/a6LnHk= github.com/buildkite/terminal-to-html/v3 v3.7.0 h1:chdLUSpiOj/A4v3dzxyOqixXI6aw7IDA6Dk77FXsvNU= github.com/buildkite/terminal-to-html/v3 v3.7.0/go.mod h1:g0ME1XqbkBSgXR9YmlIHcJIjzaMyWW+HbsG0rPb5puo= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -231,7 +236,6 @@ github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chi-middleware/proxy v1.1.1 h1:4HaXUp8o2+bhHr1OhVy+VjN0+L7/07JDcn6v7YrTjrQ= @@ -242,8 +246,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.3.2 h1:VWp8dY3yH69fdM7lM6A1+NhhVoDu9vqK0jOgmkQHFWk= -github.com/cloudflare/circl v1.3.2/go.mod h1:+CauBF6R70Jqcyl8N2hC8pAXYbWkGIezuSbuGLtRhnw= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -300,8 +304,8 @@ github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmW github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= -github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.9.0 h1:pTK/l/3qYIKaRXuHnEnIf7Y5NxfRPfpb7dis6/gdlVI= +github.com/dlclark/regexp2 v1.9.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= @@ -314,8 +318,8 @@ github.com/dvyukov/go-fuzz v0.0.0-20210429054444-fca39067bc72/go.mod h1:11Gm+ccJ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/editorconfig/editorconfig-core-go/v2 v2.5.1 h1:EMpGLI+QHJMbvppCjIFTWulZcc+lDPor4G6SRnsfRNU= -github.com/editorconfig/editorconfig-core-go/v2 v2.5.1/go.mod h1:9l0WF7U8RrFunzIpbUGLh1TIRUgDrfy0mpkyv8T7q9M= +github.com/editorconfig/editorconfig-core-go/v2 v2.5.2 h1:Z/G8cwnwOzGgTtvTutUhWPJ1ySUzuermioYMra7TuDQ= +github.com/editorconfig/editorconfig-core-go/v2 v2.5.2/go.mod h1:DoNm5QtDjTkizv0Oo1O+OJ92feoyUz3V4StZpOmP69E= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/go-bindata-assetfs v1.0.1/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/emersion/go-imap v1.2.1 h1:+s9ZjMEjOB8NzZMVTM3cCenz2JrQIGGo5j1df19WjTA= @@ -336,8 +340,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/ethantkoenig/rupture v1.0.1 h1:6aAXghmvtnngMgQzy7SMGdicMvkV86V4n9fT0meE5E4= github.com/ethantkoenig/rupture v1.0.1/go.mod h1:Sjqo/nbffZp1pVVXNGhpugIjsWmuS9KiIB4GtpEBur4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -358,8 +362,8 @@ github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-ap/activitypub v0.0.0-20230218112952-bfb607b04799 h1:zVZaYt1h4yWL7uRHvq2StewCu4ObtS+ws9gGgoZJ+2s= -github.com/go-ap/activitypub v0.0.0-20230218112952-bfb607b04799/go.mod h1:1oVD0h0aPT3OEE1ZoSUoym/UGKzxe+e0y8K2AkQ1Hqs= +github.com/go-ap/activitypub v0.0.0-20230331173947-f5b96d9450d4 h1:SGAGW21M92426IL1wW42rDHEkA2kqheNYrkFYVDNLvk= +github.com/go-ap/activitypub v0.0.0-20230331173947-f5b96d9450d4/go.mod h1:qw0WNf+PTG69Xu6mVqUluDuKl1VwVYdgntOZQFBZQ48= github.com/go-ap/errors v0.0.0-20221205040414-01c1adfc98ea h1:ywGtLGVjJjMrq4mu35Qmu+NtlhlTk/gTayE6Bb4tQZk= github.com/go-ap/errors v0.0.0-20221205040414-01c1adfc98ea/go.mod h1:SaTNjEEkp0q+w3pUS1ccyEL/lUrHteORlDq/e21mCc8= github.com/go-ap/jsonld v0.0.0-20221030091449-f2a191312c73 h1:GMKIYXyXPGIp+hYiWOhfqK4A023HdgisDT4YGgf99mw= @@ -372,36 +376,36 @@ github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= -github.com/go-enry/go-enry/v2 v2.8.3 h1:BwvNrN58JqBJhyyVdZSl5QD3xoxEEGYUrRyPh31FGhw= -github.com/go-enry/go-enry/v2 v2.8.3/go.mod h1:GVzIiAytiS5uT/QiuakK7TF1u4xDab87Y8V5EJRpsIQ= +github.com/go-enry/go-enry/v2 v2.8.4 h1:QrY3hx/RiqCJJRbdU0MOcjfTM1a586J0WSooqdlJIhs= +github.com/go-enry/go-enry/v2 v2.8.4/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI= +github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY= github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e h1:oRq/fiirun5HqlEWMLIcDmLpIELlG4iGbd0s8iqgPi8= github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= -github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw= -github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI= +github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= +github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs= github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= github.com/go-openapi/analysis v0.21.4 h1:ZDFLvSNxpDaomuCueM0BlSXxpANBlFYiBvr+GXrvIHc= github.com/go-openapi/analysis v0.21.4/go.mod h1:4zQ35W4neeZTqh3ol0rv/O8JBbka9QyAgQRPp9y3pfo= @@ -443,21 +447,20 @@ github.com/go-redis/redis/v8 v8.4.0/go.mod h1:A1tbYoHSa1fXwN+//ljcCYYJeLmVrwL9hb github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-swagger/go-swagger v0.30.4 h1:cPrWLSXY6ZdcgfRicOj0lANg72TkTHz6uv/OlUdzO5U= github.com/go-swagger/go-swagger v0.30.4/go.mod h1:YM5D5kR9c1ft3ynMXvDk2uo/7UZHKFEqKXcAL9f4Phc= github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013 h1:l9rI6sNaZgNC0LnF3MiE+qTmyBA/tZAg1rtyrGbUMK0= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= -github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= -github.com/go-testfixtures/testfixtures/v3 v3.8.1 h1:uonwvepqRvSgddcrReZQhojTlWlmOlHkYAb9ZaOMWgU= -github.com/go-testfixtures/testfixtures/v3 v3.8.1/go.mod h1:Kdu7YeMC0KRXVHdaQ91Vmx3pcjoTF63h4f1qTJDdXLA= +github.com/go-testfixtures/testfixtures/v3 v3.9.0 h1:938g5V+GWLVejm3Hc+nWCuEXRlcglZDDlN/t1gWzcSY= +github.com/go-testfixtures/testfixtures/v3 v3.9.0/go.mod h1:cdsKD2ApFBjdog9jRsz6EJqF+LClq/hrwE9K/1Dzo4s= github.com/go-webauthn/revoke v0.1.9 h1:gSJ1ckA9VaKA2GN4Ukp+kiGTk1/EXtaDb1YE8RknbS0= github.com/go-webauthn/revoke v0.1.9/go.mod h1:j6WKPnv0HovtEs++paan9g3ar46gm1NarktkXBaPR+w= -github.com/go-webauthn/webauthn v0.8.1 h1:Yv9yOxEhsJULGYLbDfEuQXtSu2RthLGzPPSN2DYdXG8= -github.com/go-webauthn/webauthn v0.8.1/go.mod h1:22OJd+TV8oHrjjXmPHtcPR82lR/yR5m5ilGiF8yPFrE= +github.com/go-webauthn/webauthn v0.8.2 h1:8KLIbpldjz9KVGHfqEgJNbkhd7bbRXhNw4QWFJE15oA= +github.com/go-webauthn/webauthn v0.8.2/go.mod h1:d+ezx/jMCNDiqSMzOchuynKb9CVU1NM9BumOnokfcVQ= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= @@ -487,13 +490,13 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/goccy/go-json v0.8.1/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.9.5/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.9.6/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= -github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogs/chardet v0.0.0-20191104214054-4b6791f73a28/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14 h1:yXtpJr/LV6PFu4nTLgfjQdcMdzjbqqXMEnHfq0Or6p8= @@ -501,7 +504,6 @@ github.com/gogs/cron v0.0.0-20171120032916-9f6c956d3e14/go.mod h1:jPoNZLWDAqA5N3 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= @@ -509,8 +511,8 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= -github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= -github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I= +github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -539,8 +541,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -588,8 +591,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/pprof v0.0.0-20230222194610-99052d3372e7 h1:pNFnpaSXfibgW7aUbk9pwLmI7LNwh/iR46x/YwN/lNg= -github.com/google/pprof v0.0.0-20230222194610-99052d3372e7/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= +github.com/google/pprof v0.0.0-20230502171905-255e3b9b56de h1:6bMcLOeKoNo0+mTOb1ee3McF6CCKGixjLR3EDQY1Jik= +github.com/google/pprof v0.0.0-20230502171905-255e3b9b56de/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -673,8 +676,9 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= +github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= @@ -691,7 +695,7 @@ github.com/jackc/pgconn v1.5.1-0.20200601181101-fa742c524853/go.mod h1:QeD3lBfpT github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.8.1/go.mod h1:JV6m6b6jhjdmzchES0drzCcYcAHS1OPD5xu3OZ/lE2g= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= -github.com/jackc/pgconn v1.12.1 h1:rsDFzIpRk7xT4B8FufgpCCeyjdNpKyghZeSefViE5W8= +github.com/jackc/pgconn v1.14.0 h1:vrbA9Ud87g6JdFWkHTJXppVce58qPIdP7N8y0Ml/A7Q= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -707,10 +711,10 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1: github.com/jackc/pgproto3/v2 v2.0.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.0 h1:brH0pCGBDkBW07HWlN/oSBXrmo3WB0UvZd1pIuDcL8Y= +github.com/jackc/pgproto3/v2 v2.3.2 h1:7eY55bdBeCz1F2fTzSz69QC+pG46jYq9/jtSPiJ5nn0= github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= @@ -719,7 +723,7 @@ github.com/jackc/pgtype v1.3.1-0.20200510190516-8cd94a14c75a/go.mod h1:vaogEUkAL github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/VzJ9/lxTO5R5sf80p0DiucVtN7ZxvaC4GmQ= github.com/jackc/pgtype v1.7.0/go.mod h1:ZnHF+rMePVqDKaOfJVI4Q8IVvAQMryDlDkZnKOI75BE= github.com/jackc/pgtype v1.8.0/go.mod h1:PqDKcEBtllAtk/2p6z6SHdXW5UB+MhE75tUol2OKexE= -github.com/jackc/pgtype v1.11.0 h1:u4uiGPz/1hryuXzyaBhSk6dnIyyG2683olG2OV+UUgs= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= @@ -728,34 +732,30 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200510190926-94ba730bb1e9/go.mod h1:t3/cdRQl6 github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9sW1JMiNn0kdYBaLelIhw7Pg4qd+Vk6tw7Hg= github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CIdYfcc= github.com/jackc/pgx/v4 v4.12.0/go.mod h1:fE547h6VulLPA3kySjfnSG/e2D861g/50JlVUa/ub60= -github.com/jackc/pgx/v4 v4.16.1 h1:JzTglcal01DrghUqt+PmzWsZx/Yh7SC/CTQmSBMTd0Y= +github.com/jackc/pgx/v4 v4.18.1 h1:YP7G1KABtKpB5IHrO9vYwSrCOhs7p3uqhvhhQBptya0= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= -github.com/jaytaylor/html2text v0.0.0-20200412013138-3577fbdbcff7/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= -github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba h1:QFQpJdgbON7I0jr2hYW7Bs+XV0qjc3d5tZoDnRFnqTg= -github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= +github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA= +github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jhillyerd/enmime v0.10.1 h1:3VP8gFhK7R948YJBrna5bOgnTXEuPAoICo79kKkBKfA= -github.com/jhillyerd/enmime v0.10.1/go.mod h1:Qpe8EEemJMFAF8+NZoWdpXvK2Yb9dRF0k/z6mkcDHsA= +github.com/jhillyerd/enmime v0.11.1 h1:U6ToGVxfxNQQhKrAaGxtwOf7Zqksb8AQ3j1CyAWOk5k= +github.com/jhillyerd/enmime v0.11.1/go.mod h1:EktNOa/V6ka9yCrfoB2uxgefp1lno6OVdszW0iQ5LnM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -764,7 +764,6 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -774,26 +773,27 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/keybase/go-crypto v0.0.0-20200123153347-de78d2cb44f4 h1:cTxwSmnaqLoo+4tLukHoB9iqHOu3LmLhRmgUxZo6Vp4= github.com/keybase/go-crypto v0.0.0-20200123153347-de78d2cb44f4/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kljensen/snowball v0.6.0/go.mod h1:27N7E8fVU5H68RlUmnWwZCfxgt4POBJfENGMvNRhldw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -816,8 +816,8 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libdns/libdns v0.2.1 h1:Wu59T7wSHRgtA0cfxC+n1c/e+O3upJGWytknkmFEDis= github.com/libdns/libdns v0.2.1/go.mod h1:yQCXzk1lEZmmCPa857bnk4TsOiqYasqpyOEeSObbb40= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -837,8 +837,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/going v1.0.0 h1:DQw0ZP7NbNlFGcKbcE/IVSOAFzScxRtLpd0rLMzLhq0= github.com/markbates/going v1.0.0/go.mod h1:I6mnB4BPnEeqo85ynXIx1ZFLLbtiLHNXVgWeFO9OGOA= -github.com/markbates/goth v1.76.0 h1:lXLpETvTJWYKnfbd9tHK/GfLFsc3ihVB8KGjfDTyIEQ= -github.com/markbates/goth v1.76.0/go.mod h1:X6xdNgpapSENS0O35iTBBcMHoJDQDfI9bJl+APCkYMc= +github.com/markbates/goth v1.77.0 h1:s3scqnWv/Zq/a5M766V0FKsLfOdFNdh/HEkuWCKbvT8= +github.com/markbates/goth v1.77.0/go.mod h1:X6xdNgpapSENS0O35iTBBcMHoJDQDfI9bJl+APCkYMc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= @@ -847,7 +847,6 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -857,36 +856,35 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/meilisearch/meilisearch-go v0.23.0 h1:CuqB+/NyEJKXF2SovTetAZW7lX+nSH+QTqbgSH6bv+Q= -github.com/meilisearch/meilisearch-go v0.23.0/go.mod h1:sAPJgywANHUCFUo/spCQ8SoP6sJhmfIKFWIXu7Dd5GQ= -github.com/mholt/acmez v1.0.4 h1:N3cE4Pek+dSolbsofIkAYz6H1d3pE+2G0os7QHslf80= -github.com/mholt/acmez v1.0.4/go.mod h1:qFGLZ4u+ehWINeJZjzPlsnjJBCPAADWTcIqE/7DAYQY= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/meilisearch/meilisearch-go v0.24.0 h1:GTP8LWZmkMYrGgX5BRZdkC2Txyp0mFYLzXYMlVV7cSQ= +github.com/meilisearch/meilisearch-go v0.24.0/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= +github.com/mholt/acmez v1.1.0 h1:IQ9CGHKOHokorxnffsqDvmmE30mDenO1lptYZ1AYkHY= +github.com/mholt/acmez v1.1.0/go.mod h1:zwo5+fbLLTowAX8o8ETfQzbDtwGEXnPhkmGdKIP+bgs= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= -github.com/microcosm-cc/bluemonday v1.0.22 h1:p2tT7RNzRdCi0qmwxG+HbqD6ILkmwter1ZwVZn1oTxA= -github.com/microcosm-cc/bluemonday v1.0.22/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= +github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= +github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.49 h1:dE5DfOtnXMXCjr/HWI6zN9vCrY6Sv666qhhiwUMvGV4= -github.com/minio/minio-go/v7 v7.0.49/go.mod h1:UI34MvQEiob3Cf/gGExGMmzugkM/tNgbFypNDy5LMVc= +github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= +github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -907,6 +905,7 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mmcloughlin/avo v0.5.0/go.mod h1:ChHFdoV7ql95Wi7vuq2YT1bwCJqiWdZrQ1im3VujLYM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -925,7 +924,6 @@ github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOl github.com/msteinert/pam v1.1.0 h1:VhLun/0n0kQYxiRBJJvVpC2jR6d21SWJFjpvUVj20Kc= github.com/msteinert/pam v1.1.0/go.mod h1:M4FPeAW8g2ITO68W8gACDz13NDJyOQM9IQsQhrR6TOI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= @@ -975,8 +973,8 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= +github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -987,6 +985,9 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/paulmach/orb v0.9.1 h1:NAL1AHClSzudQdZZA1kpjpNhcCEP1SXX+rQviOw0e/g= +github.com/paulmach/orb v0.9.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -1002,8 +1003,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pjbgf/sha1cd v0.2.3 h1:uKQP/7QOzNtKYH7UTohZLcjF5/55EnTw0jO/Ru4jZwI= -github.com/pjbgf/sha1cd v0.2.3/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1022,17 +1023,13 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= @@ -1040,33 +1037,26 @@ github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quasoft/websspi v1.1.2 h1:/mA4w0LxWlE3novvsoEL6BBA1WnjJATbjkh1kFrTidw= github.com/quasoft/websspi v1.1.2/go.mod h1:HmVdl939dQ0WIXZhyik+ARdI03M6bQzaSEKcgpFmewk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k= -github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.0.4 h1:FC82T+CHJ/Q/PdyLW++GeCO+Ol59Y4T7R4jbgjvktgc= +github.com/redis/go-redis/v9 v9.0.4/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rhysd/actionlint v1.6.23 h1:041VOXgZddfvSJa9Il+WT3Iwuo/j0Nmu4bhpAScrds4= -github.com/rhysd/actionlint v1.6.23/go.mod h1:o5qc1K3I9taGMBhL7mVkpRd64hx3YqI+3t8ewGfYXfE= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rhysd/actionlint v1.6.24 h1:5f61cF5ssP2pzG0jws5bEsfZBNhbBcO9nl7vTzVKjzs= +github.com/rhysd/actionlint v1.6.24/go.mod h1:gQmz9r2wlcpLy+VdbzK0GINJQnAK5/sNH3BpwW4Mt5I= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -1078,11 +1068,12 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -1091,16 +1082,21 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/santhosh-tekuri/jsonschema/v5 v5.2.0 h1:WCcC4vZDS1tYNxjWlwRJZQy28r8CMoggKnxNzxsVDMQ= -github.com/santhosh-tekuri/jsonschema/v5 v5.2.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.0 h1:uIkTLo0AGRc8l7h5l9r+GcYi9qfVPt6lD4/bhmzfiKo= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= +github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= +github.com/sassoftware/go-rpmutils v0.2.0/go.mod h1:TJJQYtLe/BeEmEjelI3b7xNZjzAukEkeWKmoakvaOoI= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -1112,7 +1108,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -1170,8 +1165,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= @@ -1196,14 +1192,14 @@ github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs= github.com/unknwon/com v1.0.1/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= -github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= +github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd+FkE= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= -github.com/valyala/fasthttp v1.44.0 h1:R+gLUhldIsfg1HokMuQjdQ5bh9nuXHPIfvkYUu9eR5Q= -github.com/valyala/fasthttp v1.44.0/go.mod h1:f6VbjjoI3z1NDOZOv17o6RvtRSWxC77seBFc2uWtgiY= +github.com/valyala/fasthttp v1.47.0 h1:y7moDoxYzMooFpT5aHgNgVOQDrS3qlkfiP9mDtGGK9c= +github.com/valyala/fasthttp v1.47.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= @@ -1211,8 +1207,8 @@ github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7Fw github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xanzy/go-gitlab v0.80.2 h1:CH1Q7NDklqZllox4ICVF4PwlhQGfPtE+w08Jsb74ZX0= -github.com/xanzy/go-gitlab v0.80.2/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M= +github.com/xanzy/go-gitlab v0.83.0 h1:37p0MpTPNbsTMKX/JnmJtY8Ch1sFiJzVF342+RvZEGw= +github.com/xanzy/go-gitlab v0.83.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -1231,14 +1227,13 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yohcop/openid-go v1.0.0 h1:EciJ7ZLETHR3wOtxBvKXx9RV6eyHZpCaSZ1inbBaUXE= -github.com/yohcop/openid-go v1.0.0/go.mod h1:/408xiwkeItSPJZSTPF7+VtZxPkPrRRpRNK2vjGh6yI= +github.com/yohcop/openid-go v1.0.1 h1:DPRd3iPO5F6O5zX2e62XpVAbPT6wV51cuucH0z9g3js= +github.com/yohcop/openid-go v1.0.1/go.mod h1:b/AvD03P0KHj4yuihb+VtLD6bYYgsy0zqBzPCRjkCNs= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.4.15/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= @@ -1258,8 +1253,8 @@ go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mI go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8= -go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8= +go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -1269,28 +1264,30 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opentelemetry.io/otel v0.14.0/go.mod h1:vH5xEuwy7Rts0GNtsCW3HYQoZDY+OmBJ6t1bFGGlxgw= +go.opentelemetry.io/otel v1.15.1 h1:3Iwq3lfRByPaws0f6bU3naAqOR1n5IeDWd9390kWHa8= +go.opentelemetry.io/otel v1.15.1/go.mod h1:mHHGEHVDLal6YrKMmk9LqC4a3sF5g+fHfrttQIB1NTc= +go.opentelemetry.io/otel/trace v1.15.1 h1:uXLo6iHJEzDfrNC0L0mNjItIp06SyaBQxu5t3xMlngY= +go.opentelemetry.io/otel/trace v1.15.1/go.mod h1:IWdQG/5N1x7f6YUlmdLeJvH9yxtuJAfc4VW5Agv9r/8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1309,6 +1306,7 @@ golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -1318,9 +1316,12 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1356,10 +1357,11 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1403,25 +1405,21 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210501142056-aec3718b3fa0/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= -golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1431,10 +1429,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1448,8 +1444,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1485,7 +1482,6 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1500,8 +1496,6 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1514,19 +1508,14 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210629170331-7dc0b73dc9fb/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1536,18 +1525,25 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1559,6 +1555,7 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1593,6 +1590,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1617,6 +1615,7 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1629,13 +1628,14 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1710,8 +1710,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230223222841-637eb2293923 h1:znp6mq/drrY+6khTAlJUDNFFcDGV2ENLYKpMq8SyCds= -google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1748,8 +1748,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= @@ -1779,7 +1780,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -1911,6 +1911,7 @@ modernc.org/z v1.2.19/go.mod h1:+ZpP0pc4zz97eukOzW3xagV/lS82IpPN9NGG5pNF9vY= mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/models/actions/task.go b/models/actions/task.go index ffec4c92aa8..66cd2bbea11 100644 --- a/models/actions/task.go +++ b/models/actions/task.go @@ -291,7 +291,7 @@ func CreateTaskForRunner(ctx context.Context, runner *ActionRunner) (*ActionTask } task.LogFilename = logFileName(job.Run.Repo.FullName(), task.ID) - if _, err := e.ID(task.ID).Cols("log_filename").Update(task); err != nil { + if err := UpdateTask(ctx, task, "log_filename"); err != nil { return nil, false, err } @@ -367,9 +367,18 @@ func UpdateTaskByState(ctx context.Context, state *runnerv1.TaskState) (*ActionT return nil, util.ErrNotExist } + if task.Status.IsDone() { + // the state is final, do nothing + return task, nil + } + + // state.Result is not unspecified means the task is finished if state.Result != runnerv1.Result_RESULT_UNSPECIFIED { task.Status = Status(state.Result) task.Stopped = timeutil.TimeStamp(state.StoppedAt.AsTime().Unix()) + if err := UpdateTask(ctx, task, "status", "stopped"); err != nil { + return nil, err + } if _, err := UpdateRunJob(ctx, &ActionRunJob{ ID: task.JobID, Status: task.Status, @@ -379,10 +388,6 @@ func UpdateTaskByState(ctx context.Context, state *runnerv1.TaskState) (*ActionT } } - if _, err := e.ID(task.ID).Update(task); err != nil { - return nil, err - } - if err := task.LoadAttributes(ctx); err != nil { return nil, err } @@ -440,7 +445,7 @@ func StopTask(ctx context.Context, taskID int64, status Status) error { return err } - if _, err := e.ID(task.ID).Update(task); err != nil { + if err := UpdateTask(ctx, task, "status", "stopped"); err != nil { return err } diff --git a/models/activities/action.go b/models/activities/action.go index f75ab559823..33e476e34dd 100644 --- a/models/activities/action.go +++ b/models/activities/action.go @@ -494,12 +494,27 @@ func activityQueryCondition(opts GetFeedsOptions) (builder.Cond, error) { ).From("`user`"), )) } else if !opts.Actor.IsAdmin { - cond = cond.And(builder.In("act_user_id", - builder.Select("`user`.id").Where( - builder.Eq{"keep_activity_private": false}. - And(builder.In("visibility", structs.VisibleTypePublic, structs.VisibleTypeLimited))). - Or(builder.Eq{"id": opts.Actor.ID}).From("`user`"), - )) + uidCond := builder.Select("`user`.id").From("`user`").Where( + builder.Eq{"keep_activity_private": false}. + And(builder.In("visibility", structs.VisibleTypePublic, structs.VisibleTypeLimited))). + Or(builder.Eq{"id": opts.Actor.ID}) + + if opts.RequestedUser != nil { + if opts.RequestedUser.IsOrganization() { + // An organization can always see the activities whose `act_user_id` is the same as its id. + uidCond = uidCond.Or(builder.Eq{"id": opts.RequestedUser.ID}) + } else { + // A user can always see the activities of the organizations to which the user belongs. + uidCond = uidCond.Or( + builder.Eq{"type": user_model.UserTypeOrganization}. + And(builder.In("`user`.id", builder.Select("org_id"). + Where(builder.Eq{"uid": opts.RequestedUser.ID}). + From("team_user"))), + ) + } + } + + cond = cond.And(builder.In("act_user_id", uidCond)) } // check readable repositories by doer/actor diff --git a/models/admin/task.go b/models/admin/task.go index 9e661b9997e..fc40b13d026 100644 --- a/models/admin/task.go +++ b/models/admin/task.go @@ -17,8 +17,6 @@ import ( "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" - - "xorm.io/builder" ) // Task represents a task @@ -35,7 +33,7 @@ type Task struct { StartTime timeutil.TimeStamp EndTime timeutil.TimeStamp PayloadContent string `xorm:"TEXT"` - Message string `xorm:"TEXT"` // if task failed, saved the error reason + Message string `xorm:"TEXT"` // if task failed, saved the error reason, it could be a JSON string of TranslatableMessage or a plain message Created timeutil.TimeStamp `xorm:"created"` } @@ -185,14 +183,6 @@ func GetMigratingTask(repoID int64) (*Task, error) { return &task, nil } -// HasFinishedMigratingTask returns if a finished migration task exists for the repo. -func HasFinishedMigratingTask(repoID int64) (bool, error) { - return db.GetEngine(db.DefaultContext). - Where("repo_id=? AND type=? AND status=?", repoID, structs.TaskTypeMigrateRepo, structs.TaskStatusFinished). - Table("task"). - Exist() -} - // GetMigratingTaskByID returns the migrating task by repo's id func GetMigratingTaskByID(id, doerID int64) (*Task, *migration.MigrateOptions, error) { task := Task{ @@ -214,27 +204,6 @@ func GetMigratingTaskByID(id, doerID int64) (*Task, *migration.MigrateOptions, e return &task, &opts, nil } -// FindTaskOptions find all tasks -type FindTaskOptions struct { - Status int -} - -// ToConds generates conditions for database operation. -func (opts FindTaskOptions) ToConds() builder.Cond { - cond := builder.NewCond() - if opts.Status >= 0 { - cond = cond.And(builder.Eq{"status": opts.Status}) - } - return cond -} - -// FindTasks find all tasks -func FindTasks(opts FindTaskOptions) ([]*Task, error) { - tasks := make([]*Task, 0, 10) - err := db.GetEngine(db.DefaultContext).Where(opts.ToConds()).Find(&tasks) - return tasks, err -} - // CreateTask creates a task on database func CreateTask(task *Task) error { return db.Insert(db.DefaultContext, task) diff --git a/models/asymkey/main_test.go b/models/asymkey/main_test.go index 7f8657189fd..701722be12e 100644 --- a/models/asymkey/main_test.go +++ b/models/asymkey/main_test.go @@ -8,14 +8,8 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/setting" ) -func init() { - setting.SetCustomPathAndConf("", "", "") - setting.InitProviderAndLoadCommonSettingsForTest() -} - func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ GiteaRootPath: filepath.Join("..", ".."), diff --git a/models/dbfs/main_test.go b/models/dbfs/main_test.go index 9dce663eeab..62db3592bed 100644 --- a/models/dbfs/main_test.go +++ b/models/dbfs/main_test.go @@ -8,14 +8,8 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/setting" ) -func init() { - setting.SetCustomPathAndConf("", "", "") - setting.InitProviderAndLoadCommonSettingsForTest() -} - func TestMain(m *testing.M) { unittest.MainTest(m, &unittest.TestOptions{ GiteaRootPath: filepath.Join("..", ".."), diff --git a/models/fixtures/release.yml b/models/fixtures/release.yml index 6d09401ebc4..4ed7df440db 100644 --- a/models/fixtures/release.yml +++ b/models/fixtures/release.yml @@ -108,3 +108,31 @@ is_prerelease: false is_tag: false created_unix: 946684803 + +- id: 9 + repo_id: 57 + publisher_id: 2 + tag_name: "non-existing-target-branch" + lower_tag_name: "non-existing-target-branch" + target: "non-existing" + title: "non-existing-target-branch" + sha1: "cef06e48f2642cd0dc9597b4bea09f4b3f74aad6" + num_commits: 5 + is_draft: false + is_prerelease: false + is_tag: false + created_unix: 946684803 + +- id: 10 + repo_id: 57 + publisher_id: 2 + tag_name: "empty-target-branch" + lower_tag_name: "empty-target-branch" + target: "" + title: "empty-target-branch" + sha1: "cef06e48f2642cd0dc9597b4bea09f4b3f74aad6" + num_commits: 5 + is_draft: false + is_prerelease: false + is_tag: false + created_unix: 946684803 diff --git a/models/issues/label.go b/models/issues/label.go index 35c649e8f24..9c22dcdd2d2 100644 --- a/models/issues/label.go +++ b/models/issues/label.go @@ -159,33 +159,6 @@ func (l *Label) BelongsToRepo() bool { return l.RepoID > 0 } -// Get color as RGB values in 0..255 range -func (l *Label) ColorRGB() (float64, float64, float64, error) { - color, err := strconv.ParseUint(l.Color[1:], 16, 64) - if err != nil { - return 0, 0, 0, err - } - - r := float64(uint8(0xFF & (uint32(color) >> 16))) - g := float64(uint8(0xFF & (uint32(color) >> 8))) - b := float64(uint8(0xFF & uint32(color))) - return r, g, b, nil -} - -// Determine if label text should be light or dark to be readable on background color -func (l *Label) UseLightTextColor() bool { - if strings.HasPrefix(l.Color, "#") { - if r, g, b, err := l.ColorRGB(); err == nil { - // Perceived brightness from: https://www.w3.org/TR/AERT/#color-contrast - // In the future WCAG 3 APCA may be a better solution - brightness := (0.299*r + 0.587*g + 0.114*b) / 255 - return brightness < 0.35 - } - } - - return false -} - // Return scope substring of label name, or empty string if none exists func (l *Label) ExclusiveScope() string { if !l.Exclusive { diff --git a/models/issues/label_test.go b/models/issues/label_test.go index 1f6ce4f42ee..1bc5a1a9354 100644 --- a/models/issues/label_test.go +++ b/models/issues/label_test.go @@ -22,15 +22,6 @@ func TestLabel_CalOpenIssues(t *testing.T) { assert.EqualValues(t, 2, label.NumOpenIssues) } -func TestLabel_TextColor(t *testing.T) { - assert.NoError(t, unittest.PrepareTestDatabase()) - label := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 1}) - assert.False(t, label.UseLightTextColor()) - - label = unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 2}) - assert.True(t, label.UseLightTextColor()) -} - func TestLabel_ExclusiveScope(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) label := unittest.AssertExistsAndLoadBean(t, &issues_model.Label{ID: 7}) diff --git a/models/issues/main_test.go b/models/issues/main_test.go index de84da30ecc..9fbe294f706 100644 --- a/models/issues/main_test.go +++ b/models/issues/main_test.go @@ -9,7 +9,6 @@ import ( issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/setting" _ "code.gitea.io/gitea/models" _ "code.gitea.io/gitea/models/repo" @@ -18,11 +17,6 @@ import ( "github.com/stretchr/testify/assert" ) -func init() { - setting.SetCustomPathAndConf("", "", "") - setting.InitProviderAndLoadCommonSettingsForTest() -} - func TestFixturesAreConsistent(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) unittest.CheckConsistencyFor(t, diff --git a/models/issues/pull.go b/models/issues/pull.go index a15ebec0b52..855d37867d8 100644 --- a/models/issues/pull.go +++ b/models/issues/pull.go @@ -433,6 +433,10 @@ func (pr *PullRequest) GetGitRefName() string { return fmt.Sprintf("%s%d/head", git.PullPrefix, pr.Index) } +func (pr *PullRequest) GetGitHeadBranchRefName() string { + return fmt.Sprintf("%s%s", git.BranchPrefix, pr.HeadBranch) +} + // IsChecking returns true if this pull request is still checking conflict. func (pr *PullRequest) IsChecking() bool { return pr.Status == PullRequestStatusChecking diff --git a/models/issues/pull_list.go b/models/issues/pull_list.go index 9d361c5c5da..f2adac0701d 100644 --- a/models/issues/pull_list.go +++ b/models/issues/pull_list.go @@ -51,16 +51,11 @@ func listPullRequestStatement(baseRepoID int64, opts *PullRequestsOptions) (*xor } // GetUnmergedPullRequestsByHeadInfo returns all pull requests that are open and has not been merged -// by given head information (repo and branch). -// arg `includeClosed` controls whether the SQL returns closed PRs -func GetUnmergedPullRequestsByHeadInfo(repoID int64, branch string, includeClosed bool) ([]*PullRequest, error) { +func GetUnmergedPullRequestsByHeadInfo(repoID int64, branch string) ([]*PullRequest, error) { prs := make([]*PullRequest, 0, 2) sess := db.GetEngine(db.DefaultContext). Join("INNER", "issue", "issue.id = pull_request.issue_id"). - Where("head_repo_id = ? AND head_branch = ? AND has_merged = ? AND flow = ?", repoID, branch, false, PullRequestFlowGithub) - if !includeClosed { - sess.Where("issue.is_closed = ?", false) - } + Where("head_repo_id = ? AND head_branch = ? AND has_merged = ? AND issue.is_closed = ? AND flow = ?", repoID, branch, false, false, PullRequestFlowGithub) return prs, sess.Find(&prs) } @@ -74,7 +69,7 @@ func CanMaintainerWriteToBranch(p access_model.Permission, branch string, user * return false } - prs, err := GetUnmergedPullRequestsByHeadInfo(p.Units[0].RepoID, branch, false) + prs, err := GetUnmergedPullRequestsByHeadInfo(p.Units[0].RepoID, branch) if err != nil { return false } diff --git a/models/issues/pull_test.go b/models/issues/pull_test.go index 8d99b2667ca..0b060952130 100644 --- a/models/issues/pull_test.go +++ b/models/issues/pull_test.go @@ -118,7 +118,7 @@ func TestHasUnmergedPullRequestsByHeadInfo(t *testing.T) { func TestGetUnmergedPullRequestsByHeadInfo(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - prs, err := issues_model.GetUnmergedPullRequestsByHeadInfo(1, "branch2", false) + prs, err := issues_model.GetUnmergedPullRequestsByHeadInfo(1, "branch2") assert.NoError(t, err) assert.Len(t, prs, 1) for _, pr := range prs { diff --git a/models/issues/review.go b/models/issues/review.go index fe123d73986..ed30bce149a 100644 --- a/models/issues/review.go +++ b/models/issues/review.go @@ -189,6 +189,20 @@ func (r *Review) LoadAttributes(ctx context.Context) (err error) { return err } +func (r *Review) HTMLTypeColorName() string { + switch r.Type { + case ReviewTypeApprove: + return "green" + case ReviewTypeComment: + return "grey" + case ReviewTypeReject: + return "red" + case ReviewTypeRequest: + return "yellow" + } + return "grey" +} + // GetReviewByID returns the review by the given ID func GetReviewByID(ctx context.Context, id int64) (*Review, error) { review := new(Review) diff --git a/models/main_test.go b/models/main_test.go index b5919bb2861..d490507649a 100644 --- a/models/main_test.go +++ b/models/main_test.go @@ -11,18 +11,12 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/setting" _ "code.gitea.io/gitea/models/system" "github.com/stretchr/testify/assert" ) -func init() { - setting.SetCustomPathAndConf("", "", "") - setting.InitProviderAndLoadCommonSettingsForTest() -} - // TestFixturesAreConsistent assert that test fixtures are consistent func TestFixturesAreConsistent(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) diff --git a/models/migrations/base/testlogger.go b/models/migrations/base/testlogger.go deleted file mode 100644 index 80e672952a6..00000000000 --- a/models/migrations/base/testlogger.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package base - -import ( - "context" - "fmt" - "os" - "runtime" - "strings" - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/queue" -) - -var ( - prefix string - slowTest = 10 * time.Second - slowFlush = 5 * time.Second -) - -// TestLogger is a logger which will write to the testing log -type TestLogger struct { - log.WriterLogger -} - -var writerCloser = &testLoggerWriterCloser{} - -type testLoggerWriterCloser struct { - sync.RWMutex - t []*testing.TB -} - -func (w *testLoggerWriterCloser) setT(t *testing.TB) { - w.Lock() - w.t = append(w.t, t) - w.Unlock() -} - -func (w *testLoggerWriterCloser) Write(p []byte) (int, error) { - w.RLock() - var t *testing.TB - if len(w.t) > 0 { - t = w.t[len(w.t)-1] - } - w.RUnlock() - if t != nil && *t != nil { - if len(p) > 0 && p[len(p)-1] == '\n' { - p = p[:len(p)-1] - } - - defer func() { - err := recover() - if err == nil { - return - } - var errString string - errErr, ok := err.(error) - if ok { - errString = errErr.Error() - } else { - errString, ok = err.(string) - } - if !ok { - panic(err) - } - if !strings.HasPrefix(errString, "Log in goroutine after ") { - panic(err) - } - }() - - (*t).Log(string(p)) - return len(p), nil - } - return len(p), nil -} - -func (w *testLoggerWriterCloser) Close() error { - w.Lock() - if len(w.t) > 0 { - w.t = w.t[:len(w.t)-1] - } - w.Unlock() - return nil -} - -// PrintCurrentTest prints the current test to os.Stdout -func PrintCurrentTest(t testing.TB, skip ...int) func() { - start := time.Now() - actualSkip := 1 - if len(skip) > 0 { - actualSkip = skip[0] - } - _, filename, line, _ := runtime.Caller(actualSkip) - - if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line) - } else { - fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line) - } - writerCloser.setT(&t) - return func() { - took := time.Since(start) - if took > slowTest { - if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow))) - } else { - fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", t.Name(), took) - } - } - timer := time.AfterFunc(slowFlush, func() { - if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), slowFlush) - } else { - fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), slowFlush) - } - }) - if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil { - t.Errorf("Flushing queues failed with error %v", err) - } - timer.Stop() - flushTook := time.Since(start) - took - if flushTook > slowFlush { - if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed))) - } else { - fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook) - } - } - _ = writerCloser.Close() - } -} - -// Printf takes a format and args and prints the string to os.Stdout -func Printf(format string, args ...interface{}) { - if log.CanColorStdout { - for i := 0; i < len(args); i++ { - args[i] = log.NewColoredValue(args[i]) - } - } - fmt.Fprintf(os.Stdout, "\t"+format, args...) -} - -// NewTestLogger creates a TestLogger as a log.LoggerProvider -func NewTestLogger() log.LoggerProvider { - logger := &TestLogger{} - logger.Colorize = log.CanColorStdout - logger.Level = log.TRACE - return logger -} - -// Init inits connection writer with json config. -// json config only need key "level". -func (log *TestLogger) Init(config string) error { - err := json.Unmarshal([]byte(config), log) - if err != nil { - return err - } - log.NewWriterLogger(writerCloser) - return nil -} - -// Flush when log should be flushed -func (log *TestLogger) Flush() { -} - -// ReleaseReopen does nothing -func (log *TestLogger) ReleaseReopen() error { - return nil -} - -// GetName returns the default name for this implementation -func (log *TestLogger) GetName() string { - return "test" -} diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 14f374f1a7c..08eded7fdc0 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -11,7 +11,6 @@ import ( "path" "path/filepath" "runtime" - "strings" "testing" "code.gitea.io/gitea/models/unittest" @@ -19,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/testlogger" "github.com/stretchr/testify/assert" "xorm.io/xorm" @@ -32,7 +32,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En t.Helper() ourSkip := 2 ourSkip += skip - deferFn := PrintCurrentTest(t, ourSkip) + deferFn := testlogger.PrintCurrentTest(t, ourSkip) assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) @@ -110,9 +110,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En } func MainTest(m *testing.M) { - log.Register("test", NewTestLogger) - _, filename, _, _ := runtime.Caller(0) - prefix = strings.TrimSuffix(filename, "tests/testlogger.go") + log.Register("test", testlogger.NewTestLogger) giteaRoot := base.SetupGiteaRoot() if giteaRoot == "" { @@ -150,7 +148,7 @@ func MainTest(m *testing.M) { setting.AppDataPath = tmpDataPath setting.SetCustomPathAndConf("", "", "") - setting.InitProviderAndLoadCommonSettingsForTest() + unittest.InitSettings() if err = git.InitFull(context.Background()); err != nil { fmt.Printf("Unable to InitFull: %v\n", err) os.Exit(1) diff --git a/models/packages/descriptor.go b/models/packages/descriptor.go index 9256dd56302..1cac2eb0221 100644 --- a/models/packages/descriptor.go +++ b/models/packages/descriptor.go @@ -25,6 +25,7 @@ import ( "code.gitea.io/gitea/modules/packages/nuget" "code.gitea.io/gitea/modules/packages/pub" "code.gitea.io/gitea/modules/packages/pypi" + "code.gitea.io/gitea/modules/packages/rpm" "code.gitea.io/gitea/modules/packages/rubygems" "code.gitea.io/gitea/modules/packages/swift" "code.gitea.io/gitea/modules/packages/vagrant" @@ -163,6 +164,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc metadata = &pub.Metadata{} case TypePyPI: metadata = &pypi.Metadata{} + case TypeRpm: + metadata = &rpm.VersionMetadata{} case TypeRubyGems: metadata = &rubygems.Metadata{} case TypeSwift: diff --git a/models/packages/package.go b/models/packages/package.go index cc860b9a907..a817ab6ff19 100644 --- a/models/packages/package.go +++ b/models/packages/package.go @@ -44,6 +44,7 @@ const ( TypeNuGet Type = "nuget" TypePub Type = "pub" TypePyPI Type = "pypi" + TypeRpm Type = "rpm" TypeRubyGems Type = "rubygems" TypeSwift Type = "swift" TypeVagrant Type = "vagrant" @@ -64,6 +65,7 @@ var TypeList = []Type{ TypeNuGet, TypePub, TypePyPI, + TypeRpm, TypeRubyGems, TypeSwift, TypeVagrant, @@ -100,6 +102,8 @@ func (pt Type) Name() string { return "Pub" case TypePyPI: return "PyPI" + case TypeRpm: + return "RPM" case TypeRubyGems: return "RubyGems" case TypeSwift: @@ -141,6 +145,8 @@ func (pt Type) SVGName() string { return "gitea-pub" case TypePyPI: return "gitea-python" + case TypeRpm: + return "gitea-rpm" case TypeRubyGems: return "gitea-rubygems" case TypeSwift: diff --git a/models/packages/package_file.go b/models/packages/package_file.go index 28e2a0111a5..1c2c9ac072f 100644 --- a/models/packages/package_file.go +++ b/models/packages/package_file.go @@ -118,7 +118,7 @@ func DeleteFileByID(ctx context.Context, fileID int64) error { // PackageFileSearchOptions are options for SearchXXX methods type PackageFileSearchOptions struct { OwnerID int64 - PackageType string + PackageType Type VersionID int64 Query string CompositeKey string diff --git a/models/repo.go b/models/repo.go index 5a1e2e028e8..59031328623 100644 --- a/models/repo.go +++ b/models/repo.go @@ -35,12 +35,11 @@ import ( "xorm.io/builder" ) -// ItemsPerPage maximum items per page in forks, watchers and stars of a repo -var ItemsPerPage = 40 - // Init initialize model func Init(ctx context.Context) error { - unit.LoadUnitConfig() + if err := unit.LoadUnitConfig(); err != nil { + return err + } return system_model.Init(ctx) } diff --git a/models/repo/release.go b/models/repo/release.go index 75eb27f0743..b77490584f9 100644 --- a/models/repo/release.go +++ b/models/repo/release.go @@ -72,6 +72,7 @@ type Release struct { OriginalAuthorID int64 `xorm:"index"` LowerTagName string Target string + TargetBehind string `xorm:"-"` // to handle non-existing or empty target Title string Sha1 string `xorm:"VARCHAR(40)"` NumCommits int64 diff --git a/models/repo/search.go b/models/repo/search.go new file mode 100644 index 00000000000..4d64acf8cfe --- /dev/null +++ b/models/repo/search.go @@ -0,0 +1,24 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repo + +import "code.gitea.io/gitea/models/db" + +// SearchOrderByMap represents all possible search order +var SearchOrderByMap = map[string]map[string]db.SearchOrderBy{ + "asc": { + "alpha": db.SearchOrderByAlphabetically, + "created": db.SearchOrderByOldest, + "updated": db.SearchOrderByLeastUpdated, + "size": db.SearchOrderBySize, + "id": db.SearchOrderByID, + }, + "desc": { + "alpha": db.SearchOrderByAlphabeticallyReverse, + "created": db.SearchOrderByNewest, + "updated": db.SearchOrderByRecentUpdated, + "size": db.SearchOrderBySizeReverse, + "id": db.SearchOrderByIDReverse, + }, +} diff --git a/models/unit/unit.go b/models/unit/unit.go index 3d5a8842cd6..7cd679116f1 100644 --- a/models/unit/unit.go +++ b/models/unit/unit.go @@ -4,6 +4,7 @@ package unit import ( + "errors" "fmt" "strings" @@ -106,12 +107,6 @@ var ( TypeExternalTracker, } - // MustRepoUnits contains the units could not be disabled currently - MustRepoUnits = []Type{ - TypeCode, - TypeReleases, - } - // DisabledRepoUnits contains the units that have been globally disabled DisabledRepoUnits = []Type{} ) @@ -122,18 +117,13 @@ func validateDefaultRepoUnits(defaultUnits, settingDefaultUnits []Type) []Type { // Use setting if not empty if len(settingDefaultUnits) > 0 { - // MustRepoUnits required as default - units = make([]Type, len(MustRepoUnits)) - copy(units, MustRepoUnits) + units = make([]Type, 0, len(settingDefaultUnits)) for _, settingUnit := range settingDefaultUnits { if !settingUnit.CanBeDefault() { log.Warn("Not allowed as default unit: %s", settingUnit.String()) continue } - // MustRepoUnits already added - if settingUnit.CanDisable() { - units = append(units, settingUnit) - } + units = append(units, settingUnit) } } @@ -150,30 +140,30 @@ func validateDefaultRepoUnits(defaultUnits, settingDefaultUnits []Type) []Type { } // LoadUnitConfig load units from settings -func LoadUnitConfig() { +func LoadUnitConfig() error { var invalidKeys []string DisabledRepoUnits, invalidKeys = FindUnitTypes(setting.Repository.DisabledRepoUnits...) if len(invalidKeys) > 0 { log.Warn("Invalid keys in disabled repo units: %s", strings.Join(invalidKeys, ", ")) } - // Check that must units are not disabled - for i, disabledU := range DisabledRepoUnits { - if !disabledU.CanDisable() { - log.Warn("Not allowed to global disable unit %s", disabledU.String()) - DisabledRepoUnits = append(DisabledRepoUnits[:i], DisabledRepoUnits[i+1:]...) - } - } setDefaultRepoUnits, invalidKeys := FindUnitTypes(setting.Repository.DefaultRepoUnits...) if len(invalidKeys) > 0 { log.Warn("Invalid keys in default repo units: %s", strings.Join(invalidKeys, ", ")) } DefaultRepoUnits = validateDefaultRepoUnits(DefaultRepoUnits, setDefaultRepoUnits) + if len(DefaultRepoUnits) == 0 { + return errors.New("no default repository units found") + } setDefaultForkRepoUnits, invalidKeys := FindUnitTypes(setting.Repository.DefaultForkRepoUnits...) if len(invalidKeys) > 0 { log.Warn("Invalid keys in default fork repo units: %s", strings.Join(invalidKeys, ", ")) } DefaultForkRepoUnits = validateDefaultRepoUnits(DefaultForkRepoUnits, setDefaultForkRepoUnits) + if len(DefaultForkRepoUnits) == 0 { + return errors.New("no default fork repository units found") + } + return nil } // UnitGlobalDisabled checks if unit type is global disabled @@ -186,16 +176,6 @@ func (u Type) UnitGlobalDisabled() bool { return false } -// CanDisable checks if this unit type can be disabled. -func (u *Type) CanDisable() bool { - for _, mu := range MustRepoUnits { - if *u == mu { - return false - } - } - return true -} - // CanBeDefault checks if the unit type can be a default repo unit func (u *Type) CanBeDefault() bool { for _, nadU := range NotAllowedDefaultRepoUnits { @@ -216,11 +196,6 @@ type Unit struct { MaxAccessMode perm.AccessMode // The max access mode of the unit. i.e. Read means this unit can only be read. } -// CanDisable returns if this unit could be disabled. -func (u *Unit) CanDisable() bool { - return u.Type.CanDisable() -} - // IsLessThan compares order of two units func (u Unit) IsLessThan(unit Unit) bool { if (u.Type == TypeExternalTracker || u.Type == TypeExternalWiki) && unit.Type != TypeExternalTracker && unit.Type != TypeExternalWiki { diff --git a/models/unit/unit_test.go b/models/unit/unit_test.go index 50d78171977..5c50a106bde 100644 --- a/models/unit/unit_test.go +++ b/models/unit/unit_test.go @@ -12,42 +12,84 @@ import ( ) func TestLoadUnitConfig(t *testing.T) { - defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []Type) { - DisabledRepoUnits = disabledRepoUnits - DefaultRepoUnits = defaultRepoUnits - DefaultForkRepoUnits = defaultForkRepoUnits - }(DisabledRepoUnits, DefaultRepoUnits, DefaultForkRepoUnits) - defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []string) { - setting.Repository.DisabledRepoUnits = disabledRepoUnits - setting.Repository.DefaultRepoUnits = defaultRepoUnits - setting.Repository.DefaultForkRepoUnits = defaultForkRepoUnits - }(setting.Repository.DisabledRepoUnits, setting.Repository.DefaultRepoUnits, setting.Repository.DefaultForkRepoUnits) - t.Run("regular", func(t *testing.T) { + defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []Type) { + DisabledRepoUnits = disabledRepoUnits + DefaultRepoUnits = defaultRepoUnits + DefaultForkRepoUnits = defaultForkRepoUnits + }(DisabledRepoUnits, DefaultRepoUnits, DefaultForkRepoUnits) + defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []string) { + setting.Repository.DisabledRepoUnits = disabledRepoUnits + setting.Repository.DefaultRepoUnits = defaultRepoUnits + setting.Repository.DefaultForkRepoUnits = defaultForkRepoUnits + }(setting.Repository.DisabledRepoUnits, setting.Repository.DefaultRepoUnits, setting.Repository.DefaultForkRepoUnits) + setting.Repository.DisabledRepoUnits = []string{"repo.issues"} setting.Repository.DefaultRepoUnits = []string{"repo.code", "repo.releases", "repo.issues", "repo.pulls"} setting.Repository.DefaultForkRepoUnits = []string{"repo.releases"} - LoadUnitConfig() + assert.NoError(t, LoadUnitConfig()) assert.Equal(t, []Type{TypeIssues}, DisabledRepoUnits) assert.Equal(t, []Type{TypeCode, TypeReleases, TypePullRequests}, DefaultRepoUnits) - assert.Equal(t, []Type{TypeCode, TypeReleases}, DefaultForkRepoUnits) + assert.Equal(t, []Type{TypeReleases}, DefaultForkRepoUnits) }) t.Run("invalid", func(t *testing.T) { + defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []Type) { + DisabledRepoUnits = disabledRepoUnits + DefaultRepoUnits = defaultRepoUnits + DefaultForkRepoUnits = defaultForkRepoUnits + }(DisabledRepoUnits, DefaultRepoUnits, DefaultForkRepoUnits) + defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []string) { + setting.Repository.DisabledRepoUnits = disabledRepoUnits + setting.Repository.DefaultRepoUnits = defaultRepoUnits + setting.Repository.DefaultForkRepoUnits = defaultForkRepoUnits + }(setting.Repository.DisabledRepoUnits, setting.Repository.DefaultRepoUnits, setting.Repository.DefaultForkRepoUnits) + setting.Repository.DisabledRepoUnits = []string{"repo.issues", "invalid.1"} setting.Repository.DefaultRepoUnits = []string{"repo.code", "invalid.2", "repo.releases", "repo.issues", "repo.pulls"} setting.Repository.DefaultForkRepoUnits = []string{"invalid.3", "repo.releases"} - LoadUnitConfig() + assert.NoError(t, LoadUnitConfig()) assert.Equal(t, []Type{TypeIssues}, DisabledRepoUnits) assert.Equal(t, []Type{TypeCode, TypeReleases, TypePullRequests}, DefaultRepoUnits) - assert.Equal(t, []Type{TypeCode, TypeReleases}, DefaultForkRepoUnits) + assert.Equal(t, []Type{TypeReleases}, DefaultForkRepoUnits) }) t.Run("duplicate", func(t *testing.T) { + defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []Type) { + DisabledRepoUnits = disabledRepoUnits + DefaultRepoUnits = defaultRepoUnits + DefaultForkRepoUnits = defaultForkRepoUnits + }(DisabledRepoUnits, DefaultRepoUnits, DefaultForkRepoUnits) + defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []string) { + setting.Repository.DisabledRepoUnits = disabledRepoUnits + setting.Repository.DefaultRepoUnits = defaultRepoUnits + setting.Repository.DefaultForkRepoUnits = defaultForkRepoUnits + }(setting.Repository.DisabledRepoUnits, setting.Repository.DefaultRepoUnits, setting.Repository.DefaultForkRepoUnits) + setting.Repository.DisabledRepoUnits = []string{"repo.issues", "repo.issues"} setting.Repository.DefaultRepoUnits = []string{"repo.code", "repo.releases", "repo.issues", "repo.pulls", "repo.code"} setting.Repository.DefaultForkRepoUnits = []string{"repo.releases", "repo.releases"} - LoadUnitConfig() + assert.NoError(t, LoadUnitConfig()) assert.Equal(t, []Type{TypeIssues}, DisabledRepoUnits) assert.Equal(t, []Type{TypeCode, TypeReleases, TypePullRequests}, DefaultRepoUnits) - assert.Equal(t, []Type{TypeCode, TypeReleases}, DefaultForkRepoUnits) + assert.Equal(t, []Type{TypeReleases}, DefaultForkRepoUnits) + }) + t.Run("empty_default", func(t *testing.T) { + defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []Type) { + DisabledRepoUnits = disabledRepoUnits + DefaultRepoUnits = defaultRepoUnits + DefaultForkRepoUnits = defaultForkRepoUnits + }(DisabledRepoUnits, DefaultRepoUnits, DefaultForkRepoUnits) + defer func(disabledRepoUnits, defaultRepoUnits, defaultForkRepoUnits []string) { + setting.Repository.DisabledRepoUnits = disabledRepoUnits + setting.Repository.DefaultRepoUnits = defaultRepoUnits + setting.Repository.DefaultForkRepoUnits = defaultForkRepoUnits + }(setting.Repository.DisabledRepoUnits, setting.Repository.DefaultRepoUnits, setting.Repository.DefaultForkRepoUnits) + + setting.Repository.DisabledRepoUnits = []string{"repo.issues", "repo.issues"} + setting.Repository.DefaultRepoUnits = []string{} + setting.Repository.DefaultForkRepoUnits = []string{"repo.releases", "repo.releases"} + assert.NoError(t, LoadUnitConfig()) + assert.Equal(t, []Type{TypeIssues}, DisabledRepoUnits) + assert.ElementsMatch(t, []Type{TypeCode, TypePullRequests, TypeReleases, TypeWiki, TypePackages, TypeProjects}, DefaultRepoUnits) + assert.Equal(t, []Type{TypeReleases}, DefaultForkRepoUnits) }) } diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index cff1489a7c7..10a70ad9f8e 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -6,12 +6,15 @@ package unittest import ( "context" "fmt" + "log" "os" "path/filepath" + "strings" "testing" "code.gitea.io/gitea/models/db" system_model "code.gitea.io/gitea/models/system" + "code.gitea.io/gitea/modules/auth/password/hash" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" @@ -39,6 +42,22 @@ func fatalTestError(fmtStr string, args ...interface{}) { os.Exit(1) } +// InitSettings initializes config provider and load common setttings for tests +func InitSettings(extraConfigs ...string) { + setting.Init(&setting.Options{ + AllowEmpty: true, + ExtraConfig: strings.Join(extraConfigs, "\n"), + }) + + if err := setting.PrepareAppDataPath(); err != nil { + log.Fatalf("Can not prepare APP_DATA_PATH: %v", err) + } + // register the dummy hash algorithm function used in the test fixtures + _ = hash.Register("dummy", hash.NewDummyHasher) + + setting.PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy") +} + // TestOptions represents test options type TestOptions struct { GiteaRootPath string @@ -50,6 +69,9 @@ type TestOptions struct { // MainTest a reusable TestMain(..) function for unit tests that need to use a // test database. Creates the test database, and sets necessary settings. func MainTest(m *testing.M, testOpts *TestOptions) { + setting.SetCustomPathAndConf("", "", "") + InitSettings() + var err error giteaRoot = testOpts.GiteaRootPath @@ -180,6 +202,9 @@ type FixturesOptions struct { func CreateTestEngine(opts FixturesOptions) error { x, err := xorm.NewEngine("sqlite3", "file::memory:?cache=shared&_txlock=immediate") if err != nil { + if strings.Contains(err.Error(), "unknown driver") { + return fmt.Errorf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err) + } return err } x.SetMapper(names.GonicMapper{}) diff --git a/models/webhook/hooktask.go b/models/webhook/hooktask.go index 8688fe5de36..f9fc8868263 100644 --- a/models/webhook/hooktask.go +++ b/models/webhook/hooktask.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" webhook_module "code.gitea.io/gitea/modules/webhook" gouuid "github.com/google/uuid" @@ -40,15 +41,14 @@ type HookResponse struct { // HookTask represents a hook task. type HookTask struct { - ID int64 `xorm:"pk autoincr"` - HookID int64 `xorm:"index"` - UUID string `xorm:"unique"` - api.Payloader `xorm:"-"` - PayloadContent string `xorm:"LONGTEXT"` - EventType webhook_module.HookEventType - IsDelivered bool - Delivered int64 - DeliveredString string `xorm:"-"` + ID int64 `xorm:"pk autoincr"` + HookID int64 `xorm:"index"` + UUID string `xorm:"unique"` + api.Payloader `xorm:"-"` + PayloadContent string `xorm:"LONGTEXT"` + EventType webhook_module.HookEventType + IsDelivered bool + Delivered timeutil.TimeStampNano // History info. IsSucceed bool @@ -75,8 +75,6 @@ func (t *HookTask) BeforeUpdate() { // AfterLoad updates the webhook object upon setting a column func (t *HookTask) AfterLoad() { - t.DeliveredString = time.Unix(0, t.Delivered).Format("2006-01-02 15:04:05 MST") - if len(t.RequestContent) == 0 { return } @@ -115,12 +113,17 @@ func HookTasks(hookID int64, page int) ([]*HookTask, error) { // CreateHookTask creates a new hook task, // it handles conversion from Payload to PayloadContent. func CreateHookTask(ctx context.Context, t *HookTask) (*HookTask, error) { - data, err := t.Payloader.JSONPayload() - if err != nil { - return nil, err - } t.UUID = gouuid.New().String() - t.PayloadContent = string(data) + if t.Payloader != nil { + data, err := t.Payloader.JSONPayload() + if err != nil { + return nil, err + } + t.PayloadContent = string(data) + } + if t.Delivered == 0 { + t.Delivered = timeutil.TimeStampNanoNow() + } return t, db.Insert(ctx, t) } @@ -161,13 +164,11 @@ func ReplayHookTask(ctx context.Context, hookID int64, uuid string) (*HookTask, } } - newTask := &HookTask{ - UUID: gouuid.New().String(), + return CreateHookTask(ctx, &HookTask{ HookID: task.HookID, PayloadContent: task.PayloadContent, EventType: task.EventType, - } - return newTask, db.Insert(ctx, newTask) + }) } // FindUndeliveredHookTaskIDs will find the next 100 undelivered hook tasks with ID greater than the provided lowerID diff --git a/models/webhook/webhook_test.go b/models/webhook/webhook_test.go index 74f7aeaa030..e05dcaba01a 100644 --- a/models/webhook/webhook_test.go +++ b/models/webhook/webhook_test.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/json" api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" webhook_module "code.gitea.io/gitea/modules/webhook" @@ -222,7 +223,6 @@ func TestUpdateHookTask(t *testing.T) { hook := unittest.AssertExistsAndLoadBean(t, &HookTask{ID: 1}) hook.PayloadContent = "new payload content" - hook.DeliveredString = "new delivered string" hook.IsDelivered = true unittest.AssertNotExistsBean(t, hook) assert.NoError(t, UpdateHookTask(hook)) @@ -235,7 +235,7 @@ func TestCleanupHookTaskTable_PerWebhook_DeletesDelivered(t *testing.T) { HookID: 3, Payloader: &api.PushPayload{}, IsDelivered: true, - Delivered: time.Now().UnixNano(), + Delivered: timeutil.TimeStampNanoNow(), } unittest.AssertNotExistsBean(t, hookTask) _, err := CreateHookTask(db.DefaultContext, hookTask) @@ -268,7 +268,7 @@ func TestCleanupHookTaskTable_PerWebhook_LeavesMostRecentTask(t *testing.T) { HookID: 4, Payloader: &api.PushPayload{}, IsDelivered: true, - Delivered: time.Now().UnixNano(), + Delivered: timeutil.TimeStampNanoNow(), } unittest.AssertNotExistsBean(t, hookTask) _, err := CreateHookTask(db.DefaultContext, hookTask) @@ -285,7 +285,7 @@ func TestCleanupHookTaskTable_OlderThan_DeletesDelivered(t *testing.T) { HookID: 3, Payloader: &api.PushPayload{}, IsDelivered: true, - Delivered: time.Now().AddDate(0, 0, -8).UnixNano(), + Delivered: timeutil.TimeStampNano(time.Now().AddDate(0, 0, -8).UnixNano()), } unittest.AssertNotExistsBean(t, hookTask) _, err := CreateHookTask(db.DefaultContext, hookTask) @@ -318,7 +318,7 @@ func TestCleanupHookTaskTable_OlderThan_LeavesTaskEarlierThanAgeToDelete(t *test HookID: 4, Payloader: &api.PushPayload{}, IsDelivered: true, - Delivered: time.Now().AddDate(0, 0, -6).UnixNano(), + Delivered: timeutil.TimeStampNano(time.Now().AddDate(0, 0, -6).UnixNano()), } unittest.AssertNotExistsBean(t, hookTask) _, err := CreateHookTask(db.DefaultContext, hookTask) diff --git a/modules/context/access_log.go b/modules/context/access_log.go index 64d204733b8..b6468d139bf 100644 --- a/modules/context/access_log.go +++ b/modules/context/access_log.go @@ -5,7 +5,6 @@ package context import ( "bytes" - "context" "fmt" "net" "net/http" @@ -13,8 +12,10 @@ import ( "text/template" "time" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/web/middleware" ) type routerLoggerOptions struct { @@ -26,8 +27,6 @@ type routerLoggerOptions struct { RequestID *string } -var signedUserNameStringPointerKey interface{} = "signedUserNameStringPointerKey" - const keyOfRequestIDInTemplate = ".RequestID" // According to: @@ -60,8 +59,6 @@ func AccessLogger() func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { start := time.Now() - identity := "-" - r := req.WithContext(context.WithValue(req.Context(), signedUserNameStringPointerKey, &identity)) var requestID string if needRequestID { @@ -73,9 +70,14 @@ func AccessLogger() func(http.Handler) http.Handler { reqHost = req.RemoteAddr } - next.ServeHTTP(w, r) + next.ServeHTTP(w, req) rw := w.(ResponseWriter) + identity := "-" + data := middleware.GetContextData(req.Context()) + if signedUser, ok := data[middleware.ContextDataKeySignedUser].(*user_model.User); ok { + identity = signedUser.Name + } buf := bytes.NewBuffer([]byte{}) err = logTemplate.Execute(buf, routerLoggerOptions{ req: req, diff --git a/modules/context/api.go b/modules/context/api.go index ae245ec1cb6..e263dcbe8de 100644 --- a/modules/context/api.go +++ b/modules/context/api.go @@ -222,7 +222,7 @@ func APIContexter() func(http.Handler) http.Handler { ctx := APIContext{ Context: &Context{ Resp: NewResponse(w), - Data: map[string]interface{}{}, + Data: middleware.GetContextData(req.Context()), Locale: locale, Cache: cache.GetCache(), Repo: &Repository{ @@ -250,17 +250,6 @@ func APIContexter() func(http.Handler) http.Handler { ctx.Data["Context"] = &ctx next.ServeHTTP(ctx.Resp, ctx.Req) - - // Handle adding signedUserName to the context for the AccessLogger - usernameInterface := ctx.Data["SignedUserName"] - identityPtrInterface := ctx.Req.Context().Value(signedUserNameStringPointerKey) - if usernameInterface != nil && identityPtrInterface != nil { - username := usernameInterface.(string) - identityPtr := identityPtrInterface.(*string) - if identityPtr != nil && username != "" { - *identityPtr = username - } - } }) } } diff --git a/modules/context/context.go b/modules/context/context.go index cd7fcebe55d..9ba1985f365 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -6,45 +6,28 @@ package context import ( "context" - "encoding/hex" - "errors" - "fmt" "html" "html/template" "io" - "net" "net/http" "net/url" - "path" - "strconv" "strings" "time" - "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/base" mc "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates" "code.gitea.io/gitea/modules/translation" - "code.gitea.io/gitea/modules/typesniffer" - "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web/middleware" "gitea.com/go-chi/cache" "gitea.com/go-chi/session" - chi "github.com/go-chi/chi/v5" - "github.com/minio/sha256-simd" - "golang.org/x/crypto/pbkdf2" ) -const CookieNameFlash = "gitea_flash" - // Render represents a template render type Render interface { TemplateLookup(tmpl string) (templates.TemplateExecutor, error) @@ -53,19 +36,20 @@ type Render interface { // Context represents context of a request. type Context struct { - Resp ResponseWriter - Req *http.Request - Data map[string]interface{} // data used by MVC templates - PageData map[string]interface{} // data used by JavaScript modules in one page, it's `window.config.pageData` - Render Render - translation.Locale + Resp ResponseWriter + Req *http.Request + Render Render + + Data middleware.ContextData // data used by MVC templates + PageData map[string]any // data used by JavaScript modules in one page, it's `window.config.pageData` + + Locale translation.Locale Cache cache.Cache Csrf CSRFProtector Flash *middleware.Flash Session session.Store - Link string // current request URL - EscapedLink string + Link string // current request URL (without query string) Doer *user_model.User IsSigned bool IsBasicAuth bool @@ -86,506 +70,22 @@ func (ctx *Context) Close() error { return err } -// TrHTMLEscapeArgs runs Tr but pre-escapes all arguments with html.EscapeString. +// TrHTMLEscapeArgs runs ".Locale.Tr()" but pre-escapes all arguments with html.EscapeString. // This is useful if the locale message is intended to only produce HTML content. func (ctx *Context) TrHTMLEscapeArgs(msg string, args ...string) string { trArgs := make([]interface{}, len(args)) for i, arg := range args { trArgs[i] = html.EscapeString(arg) } - return ctx.Tr(msg, trArgs...) + return ctx.Locale.Tr(msg, trArgs...) } -// GetData returns the data -func (ctx *Context) GetData() map[string]interface{} { - return ctx.Data +func (ctx *Context) Tr(msg string, args ...any) string { + return ctx.Locale.Tr(msg, args...) } -// IsUserSiteAdmin returns true if current user is a site admin -func (ctx *Context) IsUserSiteAdmin() bool { - return ctx.IsSigned && ctx.Doer.IsAdmin -} - -// IsUserRepoOwner returns true if current user owns current repo -func (ctx *Context) IsUserRepoOwner() bool { - return ctx.Repo.IsOwner() -} - -// IsUserRepoAdmin returns true if current user is admin in current repo -func (ctx *Context) IsUserRepoAdmin() bool { - return ctx.Repo.IsAdmin() -} - -// IsUserRepoWriter returns true if current user has write privilege in current repo -func (ctx *Context) IsUserRepoWriter(unitTypes []unit.Type) bool { - for _, unitType := range unitTypes { - if ctx.Repo.CanWrite(unitType) { - return true - } - } - - return false -} - -// IsUserRepoReaderSpecific returns true if current user can read current repo's specific part -func (ctx *Context) IsUserRepoReaderSpecific(unitType unit.Type) bool { - return ctx.Repo.CanRead(unitType) -} - -// IsUserRepoReaderAny returns true if current user can read any part of current repo -func (ctx *Context) IsUserRepoReaderAny() bool { - return ctx.Repo.HasAccess() -} - -// RedirectToUser redirect to a differently-named user -func RedirectToUser(ctx *Context, userName string, redirectUserID int64) { - user, err := user_model.GetUserByID(ctx, redirectUserID) - if err != nil { - ctx.ServerError("GetUserByID", err) - return - } - - redirectPath := strings.Replace( - ctx.Req.URL.EscapedPath(), - url.PathEscape(userName), - url.PathEscape(user.Name), - 1, - ) - if ctx.Req.URL.RawQuery != "" { - redirectPath += "?" + ctx.Req.URL.RawQuery - } - ctx.Redirect(path.Join(setting.AppSubURL, redirectPath), http.StatusTemporaryRedirect) -} - -// HasAPIError returns true if error occurs in form validation. -func (ctx *Context) HasAPIError() bool { - hasErr, ok := ctx.Data["HasError"] - if !ok { - return false - } - return hasErr.(bool) -} - -// GetErrMsg returns error message -func (ctx *Context) GetErrMsg() string { - return ctx.Data["ErrorMsg"].(string) -} - -// HasError returns true if error occurs in form validation. -// Attention: this function changes ctx.Data and ctx.Flash -func (ctx *Context) HasError() bool { - hasErr, ok := ctx.Data["HasError"] - if !ok { - return false - } - ctx.Flash.ErrorMsg = ctx.Data["ErrorMsg"].(string) - ctx.Data["Flash"] = ctx.Flash - return hasErr.(bool) -} - -// HasValue returns true if value of given name exists. -func (ctx *Context) HasValue(name string) bool { - _, ok := ctx.Data[name] - return ok -} - -// RedirectToFirst redirects to first not empty URL -func (ctx *Context) RedirectToFirst(location ...string) { - for _, loc := range location { - if len(loc) == 0 { - continue - } - - // Unfortunately browsers consider a redirect Location with preceding "//" and "/\" as meaning redirect to "http(s)://REST_OF_PATH" - // Therefore we should ignore these redirect locations to prevent open redirects - if len(loc) > 1 && loc[0] == '/' && (loc[1] == '/' || loc[1] == '\\') { - continue - } - - u, err := url.Parse(loc) - if err != nil || ((u.Scheme != "" || u.Host != "") && !strings.HasPrefix(strings.ToLower(loc), strings.ToLower(setting.AppURL))) { - continue - } - - ctx.Redirect(loc) - return - } - - ctx.Redirect(setting.AppSubURL + "/") -} - -const tplStatus500 base.TplName = "status/500" - -// HTML calls Context.HTML and renders the template to HTTP response -func (ctx *Context) HTML(status int, name base.TplName) { - log.Debug("Template: %s", name) - tmplStartTime := time.Now() - if !setting.IsProd { - ctx.Data["TemplateName"] = name - } - ctx.Data["TemplateLoadTimes"] = func() string { - return strconv.FormatInt(time.Since(tmplStartTime).Nanoseconds()/1e6, 10) + "ms" - } - if err := ctx.Render.HTML(ctx.Resp, status, string(name), templates.BaseVars().Merge(ctx.Data)); err != nil { - if status == http.StatusInternalServerError && name == tplStatus500 { - ctx.PlainText(http.StatusInternalServerError, "Unable to find HTML templates, the template system is not initialized, or Gitea can't find your template files.") - return - } - err = fmt.Errorf("failed to render template: %s, error: %s", name, templates.HandleTemplateRenderingError(err)) - ctx.ServerError("Render failed", err) - } -} - -// RenderToString renders the template content to a string -func (ctx *Context) RenderToString(name base.TplName, data map[string]interface{}) (string, error) { - var buf strings.Builder - err := ctx.Render.HTML(&buf, http.StatusOK, string(name), data) - return buf.String(), err -} - -// RenderWithErr used for page has form validation but need to prompt error to users. -func (ctx *Context) RenderWithErr(msg string, tpl base.TplName, form interface{}) { - if form != nil { - middleware.AssignForm(form, ctx.Data) - } - ctx.Flash.ErrorMsg = msg - ctx.Data["Flash"] = ctx.Flash - ctx.HTML(http.StatusOK, tpl) -} - -// NotFound displays a 404 (Not Found) page and prints the given error, if any. -func (ctx *Context) NotFound(logMsg string, logErr error) { - ctx.notFoundInternal(logMsg, logErr) -} - -func (ctx *Context) notFoundInternal(logMsg string, logErr error) { - if logErr != nil { - log.Log(2, log.DEBUG, "%s: %v", logMsg, logErr) - if !setting.IsProd { - ctx.Data["ErrorMsg"] = logErr - } - } - - // response simple message if Accept isn't text/html - showHTML := false - for _, part := range ctx.Req.Header["Accept"] { - if strings.Contains(part, "text/html") { - showHTML = true - break - } - } - - if !showHTML { - ctx.plainTextInternal(3, http.StatusNotFound, []byte("Not found.\n")) - return - } - - ctx.Data["IsRepo"] = ctx.Repo.Repository != nil - ctx.Data["Title"] = "Page Not Found" - ctx.HTML(http.StatusNotFound, base.TplName("status/404")) -} - -// ServerError displays a 500 (Internal Server Error) page and prints the given error, if any. -func (ctx *Context) ServerError(logMsg string, logErr error) { - ctx.serverErrorInternal(logMsg, logErr) -} - -func (ctx *Context) serverErrorInternal(logMsg string, logErr error) { - if logErr != nil { - log.ErrorWithSkip(2, "%s: %v", logMsg, logErr) - if _, ok := logErr.(*net.OpError); ok || errors.Is(logErr, &net.OpError{}) { - // This is an error within the underlying connection - // and further rendering will not work so just return - return - } - - // it's safe to show internal error to admin users, and it helps - if !setting.IsProd || (ctx.Doer != nil && ctx.Doer.IsAdmin) { - ctx.Data["ErrorMsg"] = fmt.Sprintf("%s, %s", logMsg, logErr) - } - } - - ctx.Data["Title"] = "Internal Server Error" - ctx.HTML(http.StatusInternalServerError, tplStatus500) -} - -// NotFoundOrServerError use error check function to determine if the error -// is about not found. It responds with 404 status code for not found error, -// or error context description for logging purpose of 500 server error. -func (ctx *Context) NotFoundOrServerError(logMsg string, errCheck func(error) bool, logErr error) { - if errCheck(logErr) { - ctx.notFoundInternal(logMsg, logErr) - return - } - ctx.serverErrorInternal(logMsg, logErr) -} - -// PlainTextBytes renders bytes as plain text -func (ctx *Context) plainTextInternal(skip, status int, bs []byte) { - statusPrefix := status / 100 - if statusPrefix == 4 || statusPrefix == 5 { - log.Log(skip, log.TRACE, "plainTextInternal (status=%d): %s", status, string(bs)) - } - ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") - ctx.Resp.Header().Set("X-Content-Type-Options", "nosniff") - ctx.Resp.WriteHeader(status) - if _, err := ctx.Resp.Write(bs); err != nil { - log.ErrorWithSkip(skip, "plainTextInternal (status=%d): write bytes failed: %v", status, err) - } -} - -// PlainTextBytes renders bytes as plain text -func (ctx *Context) PlainTextBytes(status int, bs []byte) { - ctx.plainTextInternal(2, status, bs) -} - -// PlainText renders content as plain text -func (ctx *Context) PlainText(status int, text string) { - ctx.plainTextInternal(2, status, []byte(text)) -} - -// RespHeader returns the response header -func (ctx *Context) RespHeader() http.Header { - return ctx.Resp.Header() -} - -type ServeHeaderOptions struct { - ContentType string // defaults to "application/octet-stream" - ContentTypeCharset string - ContentLength *int64 - Disposition string // defaults to "attachment" - Filename string - CacheDuration time.Duration // defaults to 5 minutes - LastModified time.Time -} - -// SetServeHeaders sets necessary content serve headers -func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { - header := ctx.Resp.Header() - - contentType := typesniffer.ApplicationOctetStream - if opts.ContentType != "" { - if opts.ContentTypeCharset != "" { - contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset) - } else { - contentType = opts.ContentType - } - } - header.Set("Content-Type", contentType) - header.Set("X-Content-Type-Options", "nosniff") - - if opts.ContentLength != nil { - header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10)) - } - - if opts.Filename != "" { - disposition := opts.Disposition - if disposition == "" { - disposition = "attachment" - } - - backslashEscapedName := strings.ReplaceAll(strings.ReplaceAll(opts.Filename, `\`, `\\`), `"`, `\"`) // \ -> \\, " -> \" - header.Set("Content-Disposition", fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, backslashEscapedName, url.PathEscape(opts.Filename))) - header.Set("Access-Control-Expose-Headers", "Content-Disposition") - } - - duration := opts.CacheDuration - if duration == 0 { - duration = 5 * time.Minute - } - httpcache.SetCacheControlInHeader(header, duration) - - if !opts.LastModified.IsZero() { - header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat)) - } -} - -// ServeContent serves content to http request -func (ctx *Context) ServeContent(r io.ReadSeeker, opts *ServeHeaderOptions) { - ctx.SetServeHeaders(opts) - http.ServeContent(ctx.Resp, ctx.Req, opts.Filename, opts.LastModified, r) -} - -// UploadStream returns the request body or the first form file -// Only form files need to get closed. -func (ctx *Context) UploadStream() (rd io.ReadCloser, needToClose bool, err error) { - contentType := strings.ToLower(ctx.Req.Header.Get("Content-Type")) - if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") || strings.HasPrefix(contentType, "multipart/form-data") { - if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil { - return nil, false, err - } - if ctx.Req.MultipartForm.File == nil { - return nil, false, http.ErrMissingFile - } - for _, files := range ctx.Req.MultipartForm.File { - if len(files) > 0 { - r, err := files[0].Open() - return r, true, err - } - } - return nil, false, http.ErrMissingFile - } - return ctx.Req.Body, false, nil -} - -// Error returned an error to web browser -func (ctx *Context) Error(status int, contents ...string) { - v := http.StatusText(status) - if len(contents) > 0 { - v = contents[0] - } - http.Error(ctx.Resp, v, status) -} - -// JSON render content as JSON -func (ctx *Context) JSON(status int, content interface{}) { - ctx.Resp.Header().Set("Content-Type", "application/json;charset=utf-8") - ctx.Resp.WriteHeader(status) - if err := json.NewEncoder(ctx.Resp).Encode(content); err != nil { - ctx.ServerError("Render JSON failed", err) - } -} - -func removeSessionCookieHeader(w http.ResponseWriter) { - cookies := w.Header()["Set-Cookie"] - w.Header().Del("Set-Cookie") - for _, cookie := range cookies { - if strings.HasPrefix(cookie, setting.SessionConfig.CookieName+"=") { - continue - } - w.Header().Add("Set-Cookie", cookie) - } -} - -// Redirect redirects the request -func (ctx *Context) Redirect(location string, status ...int) { - code := http.StatusSeeOther - if len(status) == 1 { - code = status[0] - } - - if strings.Contains(location, "://") || strings.HasPrefix(location, "//") { - // Some browsers (Safari) have buggy behavior for Cookie + Cache + External Redirection, eg: /my-path => https://other/path - // 1. the first request to "/my-path" contains cookie - // 2. some time later, the request to "/my-path" doesn't contain cookie (caused by Prevent web tracking) - // 3. Gitea's Sessioner doesn't see the session cookie, so it generates a new session id, and returns it to browser - // 4. then the browser accepts the empty session, then the user is logged out - // So in this case, we should remove the session cookie from the response header - removeSessionCookieHeader(ctx.Resp) - } - http.Redirect(ctx.Resp, ctx.Req, location, code) -} - -// SetSiteCookie convenience function to set most cookies consistently -// CSRF and a few others are the exception here -func (ctx *Context) SetSiteCookie(name, value string, maxAge int) { - middleware.SetSiteCookie(ctx.Resp, name, value, maxAge) -} - -// DeleteSiteCookie convenience function to delete most cookies consistently -// CSRF and a few others are the exception here -func (ctx *Context) DeleteSiteCookie(name string) { - middleware.SetSiteCookie(ctx.Resp, name, "", -1) -} - -// GetSiteCookie returns given cookie value from request header. -func (ctx *Context) GetSiteCookie(name string) string { - return middleware.GetSiteCookie(ctx.Req, name) -} - -// GetSuperSecureCookie returns given cookie value from request header with secret string. -func (ctx *Context) GetSuperSecureCookie(secret, name string) (string, bool) { - val := ctx.GetSiteCookie(name) - return ctx.CookieDecrypt(secret, val) -} - -// CookieDecrypt returns given value from with secret string. -func (ctx *Context) CookieDecrypt(secret, val string) (string, bool) { - if val == "" { - return "", false - } - - text, err := hex.DecodeString(val) - if err != nil { - return "", false - } - - key := pbkdf2.Key([]byte(secret), []byte(secret), 1000, 16, sha256.New) - text, err = util.AESGCMDecrypt(key, text) - return string(text), err == nil -} - -// SetSuperSecureCookie sets given cookie value to response header with secret string. -func (ctx *Context) SetSuperSecureCookie(secret, name, value string, maxAge int) { - text := ctx.CookieEncrypt(secret, value) - ctx.SetSiteCookie(name, text, maxAge) -} - -// CookieEncrypt encrypts a given value using the provided secret -func (ctx *Context) CookieEncrypt(secret, value string) string { - key := pbkdf2.Key([]byte(secret), []byte(secret), 1000, 16, sha256.New) - text, err := util.AESGCMEncrypt(key, []byte(value)) - if err != nil { - panic("error encrypting cookie: " + err.Error()) - } - - return hex.EncodeToString(text) -} - -// GetCookieInt returns cookie result in int type. -func (ctx *Context) GetCookieInt(name string) int { - r, _ := strconv.Atoi(ctx.GetSiteCookie(name)) - return r -} - -// GetCookieInt64 returns cookie result in int64 type. -func (ctx *Context) GetCookieInt64(name string) int64 { - r, _ := strconv.ParseInt(ctx.GetSiteCookie(name), 10, 64) - return r -} - -// GetCookieFloat64 returns cookie result in float64 type. -func (ctx *Context) GetCookieFloat64(name string) float64 { - v, _ := strconv.ParseFloat(ctx.GetSiteCookie(name), 64) - return v -} - -// RemoteAddr returns the client machie ip address -func (ctx *Context) RemoteAddr() string { - return ctx.Req.RemoteAddr -} - -// Params returns the param on route -func (ctx *Context) Params(p string) string { - s, _ := url.PathUnescape(chi.URLParam(ctx.Req, strings.TrimPrefix(p, ":"))) - return s -} - -// ParamsInt64 returns the param on route as int64 -func (ctx *Context) ParamsInt64(p string) int64 { - v, _ := strconv.ParseInt(ctx.Params(p), 10, 64) - return v -} - -// SetParams set params into routes -func (ctx *Context) SetParams(k, v string) { - chiCtx := chi.RouteContext(ctx) - chiCtx.URLParams.Add(strings.TrimPrefix(k, ":"), url.PathEscape(v)) -} - -// Write writes data to web browser -func (ctx *Context) Write(bs []byte) (int, error) { - return ctx.Resp.Write(bs) -} - -// Written returns true if there are something sent to web browser -func (ctx *Context) Written() bool { - return ctx.Resp.Status() > 0 -} - -// Status writes status code -func (ctx *Context) Status(status int) { - ctx.Resp.WriteHeader(status) +func (ctx *Context) TrN(cnt any, key1, keyN string, args ...any) string { + return ctx.Locale.TrN(cnt, key1, keyN, args...) } // Deadline is part of the interface for context.Context and we pass this to the request context @@ -614,25 +114,6 @@ func (ctx *Context) Value(key interface{}) interface{} { return ctx.Req.Context().Value(key) } -// SetTotalCountHeader set "X-Total-Count" header -func (ctx *Context) SetTotalCountHeader(total int64) { - ctx.RespHeader().Set("X-Total-Count", fmt.Sprint(total)) - ctx.AppendAccessControlExposeHeaders("X-Total-Count") -} - -// AppendAccessControlExposeHeaders append headers by name to "Access-Control-Expose-Headers" header -func (ctx *Context) AppendAccessControlExposeHeaders(names ...string) { - val := ctx.RespHeader().Get("Access-Control-Expose-Headers") - if len(val) != 0 { - ctx.RespHeader().Set("Access-Control-Expose-Headers", fmt.Sprintf("%s, %s", val, strings.Join(names, ", "))) - } else { - ctx.RespHeader().Set("Access-Control-Expose-Headers", strings.Join(names, ", ")) - } -} - -// Handler represents a custom handler -type Handler func(*Context) - type contextKeyType struct{} var contextKey interface{} = contextKeyType{} @@ -650,19 +131,10 @@ func GetContext(req *http.Request) *Context { return nil } -// GetContextUser returns context user -func GetContextUser(req *http.Request) *user_model.User { - if apiContext, ok := req.Context().Value(apiContextKey).(*APIContext); ok { - return apiContext.Doer - } - if ctx, ok := req.Context().Value(contextKey).(*Context); ok { - return ctx.Doer - } - return nil -} - -func getCsrfOpts() CsrfOptions { - return CsrfOptions{ +// Contexter initializes a classic context for a request. +func Contexter() func(next http.Handler) http.Handler { + rnd := templates.HTMLRenderer() + csrfOpts := CsrfOptions{ Secret: setting.SecretKey, Cookie: setting.CSRFCookieName, SetCookie: true, @@ -673,45 +145,35 @@ func getCsrfOpts() CsrfOptions { CookiePath: setting.SessionConfig.CookiePath, SameSite: setting.SessionConfig.SameSite, } -} - -// Contexter initializes a classic context for a request. -func Contexter(ctx context.Context) func(next http.Handler) http.Handler { - rnd := templates.HTMLRenderer() - csrfOpts := getCsrfOpts() if !setting.IsProd { CsrfTokenRegenerationInterval = 5 * time.Second // in dev, re-generate the tokens more aggressively for debug purpose } return func(next http.Handler) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - locale := middleware.Locale(resp, req) - startTime := time.Now() - link := setting.AppSubURL + strings.TrimSuffix(req.URL.EscapedPath(), "/") - ctx := Context{ Resp: NewResponse(resp), Cache: mc.GetCache(), - Locale: locale, - Link: link, + Locale: middleware.Locale(resp, req), + Link: setting.AppSubURL + strings.TrimSuffix(req.URL.EscapedPath(), "/"), Render: rnd, Session: session.GetSession(req), Repo: &Repository{ PullRequest: &PullRequest{}, }, - Org: &Organization{}, - Data: map[string]interface{}{ - "CurrentURL": setting.AppSubURL + req.URL.RequestURI(), - "PageStartTime": startTime, - "Link": link, - "RunModeIsProd": setting.IsProd, - }, + Org: &Organization{}, + Data: middleware.GetContextData(req.Context()), } defer ctx.Close() - // PageData is passed by reference, and it will be rendered to `window.config.pageData` in `head.tmpl` for JavaScript modules - ctx.PageData = map[string]interface{}{} - ctx.Data["PageData"] = ctx.PageData + ctx.Data.MergeFrom(middleware.CommonTemplateContextData()) ctx.Data["Context"] = &ctx + ctx.Data["CurrentURL"] = setting.AppSubURL + req.URL.RequestURI() + ctx.Data["Link"] = ctx.Link + ctx.Data["locale"] = ctx.Locale + + // PageData is passed by reference, and it will be rendered to `window.config.pageData` in `head.tmpl` for JavaScript modules + ctx.PageData = map[string]any{} + ctx.Data["PageData"] = ctx.PageData ctx.Req = WithContext(req, &ctx) ctx.Csrf = PrepareCSRFProtector(csrfOpts, &ctx) @@ -755,16 +217,6 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler { ctx.Data["CsrfTokenHtml"] = template.HTML(``) // FIXME: do we really always need these setting? There should be someway to have to avoid having to always set these - ctx.Data["IsLandingPageHome"] = setting.LandingPageURL == setting.LandingPageHome - ctx.Data["IsLandingPageExplore"] = setting.LandingPageURL == setting.LandingPageExplore - ctx.Data["IsLandingPageOrganizations"] = setting.LandingPageURL == setting.LandingPageOrganizations - - ctx.Data["ShowRegistrationButton"] = setting.Service.ShowRegistrationButton - ctx.Data["ShowMilestonesDashboardPage"] = setting.Service.ShowMilestonesDashboardPage - ctx.Data["ShowFooterVersion"] = setting.Other.ShowFooterVersion - - ctx.Data["EnableSwagger"] = setting.API.EnableSwagger - ctx.Data["EnableOpenIDSignIn"] = setting.Service.EnableOpenIDSignIn ctx.Data["DisableMigrations"] = setting.Repository.DisableMigrations ctx.Data["DisableStars"] = setting.Repository.DisableStars ctx.Data["EnableActions"] = setting.Actions.Enabled @@ -777,39 +229,9 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler { ctx.Data["UnitProjectsGlobalDisabled"] = unit.TypeProjects.UnitGlobalDisabled() ctx.Data["UnitActionsGlobalDisabled"] = unit.TypeActions.UnitGlobalDisabled() - ctx.Data["locale"] = locale ctx.Data["AllLangs"] = translation.AllLangs() next.ServeHTTP(ctx.Resp, ctx.Req) - - // Handle adding signedUserName to the context for the AccessLogger - usernameInterface := ctx.Data["SignedUserName"] - identityPtrInterface := ctx.Req.Context().Value(signedUserNameStringPointerKey) - if usernameInterface != nil && identityPtrInterface != nil { - username := usernameInterface.(string) - identityPtr := identityPtrInterface.(*string) - if identityPtr != nil && username != "" { - *identityPtr = username - } - } }) } } - -// SearchOrderByMap represents all possible search order -var SearchOrderByMap = map[string]map[string]db.SearchOrderBy{ - "asc": { - "alpha": db.SearchOrderByAlphabetically, - "created": db.SearchOrderByOldest, - "updated": db.SearchOrderByLeastUpdated, - "size": db.SearchOrderBySize, - "id": db.SearchOrderByID, - }, - "desc": { - "alpha": db.SearchOrderByAlphabeticallyReverse, - "created": db.SearchOrderByNewest, - "updated": db.SearchOrderByRecentUpdated, - "size": db.SearchOrderBySizeReverse, - "id": db.SearchOrderByIDReverse, - }, -} diff --git a/modules/context/context_cookie.go b/modules/context/context_cookie.go new file mode 100644 index 00000000000..9ce67a52981 --- /dev/null +++ b/modules/context/context_cookie.go @@ -0,0 +1,86 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package context + +import ( + "encoding/hex" + "net/http" + "strings" + + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/web/middleware" + + "github.com/minio/sha256-simd" + "golang.org/x/crypto/pbkdf2" +) + +const CookieNameFlash = "gitea_flash" + +func removeSessionCookieHeader(w http.ResponseWriter) { + cookies := w.Header()["Set-Cookie"] + w.Header().Del("Set-Cookie") + for _, cookie := range cookies { + if strings.HasPrefix(cookie, setting.SessionConfig.CookieName+"=") { + continue + } + w.Header().Add("Set-Cookie", cookie) + } +} + +// SetSiteCookie convenience function to set most cookies consistently +// CSRF and a few others are the exception here +func (ctx *Context) SetSiteCookie(name, value string, maxAge int) { + middleware.SetSiteCookie(ctx.Resp, name, value, maxAge) +} + +// DeleteSiteCookie convenience function to delete most cookies consistently +// CSRF and a few others are the exception here +func (ctx *Context) DeleteSiteCookie(name string) { + middleware.SetSiteCookie(ctx.Resp, name, "", -1) +} + +// GetSiteCookie returns given cookie value from request header. +func (ctx *Context) GetSiteCookie(name string) string { + return middleware.GetSiteCookie(ctx.Req, name) +} + +// GetSuperSecureCookie returns given cookie value from request header with secret string. +func (ctx *Context) GetSuperSecureCookie(secret, name string) (string, bool) { + val := ctx.GetSiteCookie(name) + return ctx.CookieDecrypt(secret, val) +} + +// CookieDecrypt returns given value from with secret string. +func (ctx *Context) CookieDecrypt(secret, val string) (string, bool) { + if val == "" { + return "", false + } + + text, err := hex.DecodeString(val) + if err != nil { + return "", false + } + + key := pbkdf2.Key([]byte(secret), []byte(secret), 1000, 16, sha256.New) + text, err = util.AESGCMDecrypt(key, text) + return string(text), err == nil +} + +// SetSuperSecureCookie sets given cookie value to response header with secret string. +func (ctx *Context) SetSuperSecureCookie(secret, name, value string, maxAge int) { + text := ctx.CookieEncrypt(secret, value) + ctx.SetSiteCookie(name, text, maxAge) +} + +// CookieEncrypt encrypts a given value using the provided secret +func (ctx *Context) CookieEncrypt(secret, value string) string { + key := pbkdf2.Key([]byte(secret), []byte(secret), 1000, 16, sha256.New) + text, err := util.AESGCMEncrypt(key, []byte(value)) + if err != nil { + panic("error encrypting cookie: " + err.Error()) + } + + return hex.EncodeToString(text) +} diff --git a/modules/context/context_data.go b/modules/context/context_data.go new file mode 100644 index 00000000000..cdf4ff9afe3 --- /dev/null +++ b/modules/context/context_data.go @@ -0,0 +1,43 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package context + +import "code.gitea.io/gitea/modules/web/middleware" + +// GetData returns the data +func (ctx *Context) GetData() middleware.ContextData { + return ctx.Data +} + +// HasAPIError returns true if error occurs in form validation. +func (ctx *Context) HasAPIError() bool { + hasErr, ok := ctx.Data["HasError"] + if !ok { + return false + } + return hasErr.(bool) +} + +// GetErrMsg returns error message +func (ctx *Context) GetErrMsg() string { + return ctx.Data["ErrorMsg"].(string) +} + +// HasError returns true if error occurs in form validation. +// Attention: this function changes ctx.Data and ctx.Flash +func (ctx *Context) HasError() bool { + hasErr, ok := ctx.Data["HasError"] + if !ok { + return false + } + ctx.Flash.ErrorMsg = ctx.Data["ErrorMsg"].(string) + ctx.Data["Flash"] = ctx.Flash + return hasErr.(bool) +} + +// HasValue returns true if value of given name exists. +func (ctx *Context) HasValue(name string) bool { + _, ok := ctx.Data[name] + return ok +} diff --git a/modules/context/form.go b/modules/context/context_form.go similarity index 93% rename from modules/context/form.go rename to modules/context/context_form.go index f9c4ab6a984..5c021525822 100644 --- a/modules/context/form.go +++ b/modules/context/context_form.go @@ -65,3 +65,8 @@ func (ctx *Context) FormOptionalBool(key string) util.OptionalBool { v = v || strings.EqualFold(s, "on") return util.OptionalBoolOf(v) } + +func (ctx *Context) SetFormString(key, value string) { + _ = ctx.Req.FormValue(key) // force parse form + ctx.Req.Form.Set(key, value) +} diff --git a/modules/context/context_model.go b/modules/context/context_model.go new file mode 100644 index 00000000000..4f70aac5169 --- /dev/null +++ b/modules/context/context_model.go @@ -0,0 +1,29 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package context + +import ( + "code.gitea.io/gitea/models/unit" +) + +// IsUserSiteAdmin returns true if current user is a site admin +func (ctx *Context) IsUserSiteAdmin() bool { + return ctx.IsSigned && ctx.Doer.IsAdmin +} + +// IsUserRepoAdmin returns true if current user is admin in current repo +func (ctx *Context) IsUserRepoAdmin() bool { + return ctx.Repo.IsAdmin() +} + +// IsUserRepoWriter returns true if current user has write privilege in current repo +func (ctx *Context) IsUserRepoWriter(unitTypes []unit.Type) bool { + for _, unitType := range unitTypes { + if ctx.Repo.CanWrite(unitType) { + return true + } + } + + return false +} diff --git a/modules/context/context_request.go b/modules/context/context_request.go new file mode 100644 index 00000000000..0b87552c085 --- /dev/null +++ b/modules/context/context_request.go @@ -0,0 +1,59 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package context + +import ( + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/go-chi/chi/v5" +) + +// RemoteAddr returns the client machine ip address +func (ctx *Context) RemoteAddr() string { + return ctx.Req.RemoteAddr +} + +// Params returns the param on route +func (ctx *Context) Params(p string) string { + s, _ := url.PathUnescape(chi.URLParam(ctx.Req, strings.TrimPrefix(p, ":"))) + return s +} + +// ParamsInt64 returns the param on route as int64 +func (ctx *Context) ParamsInt64(p string) int64 { + v, _ := strconv.ParseInt(ctx.Params(p), 10, 64) + return v +} + +// SetParams set params into routes +func (ctx *Context) SetParams(k, v string) { + chiCtx := chi.RouteContext(ctx) + chiCtx.URLParams.Add(strings.TrimPrefix(k, ":"), url.PathEscape(v)) +} + +// UploadStream returns the request body or the first form file +// Only form files need to get closed. +func (ctx *Context) UploadStream() (rd io.ReadCloser, needToClose bool, err error) { + contentType := strings.ToLower(ctx.Req.Header.Get("Content-Type")) + if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") || strings.HasPrefix(contentType, "multipart/form-data") { + if err := ctx.Req.ParseMultipartForm(32 << 20); err != nil { + return nil, false, err + } + if ctx.Req.MultipartForm.File == nil { + return nil, false, http.ErrMissingFile + } + for _, files := range ctx.Req.MultipartForm.File { + if len(files) > 0 { + r, err := files[0].Open() + return r, true, err + } + } + return nil, false, http.ErrMissingFile + } + return ctx.Req.Body, false, nil +} diff --git a/modules/context/context_response.go b/modules/context/context_response.go new file mode 100644 index 00000000000..8adff96994b --- /dev/null +++ b/modules/context/context_response.go @@ -0,0 +1,279 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package context + +import ( + "errors" + "fmt" + "net" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" + + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/templates" + "code.gitea.io/gitea/modules/web/middleware" +) + +// SetTotalCountHeader set "X-Total-Count" header +func (ctx *Context) SetTotalCountHeader(total int64) { + ctx.RespHeader().Set("X-Total-Count", fmt.Sprint(total)) + ctx.AppendAccessControlExposeHeaders("X-Total-Count") +} + +// AppendAccessControlExposeHeaders append headers by name to "Access-Control-Expose-Headers" header +func (ctx *Context) AppendAccessControlExposeHeaders(names ...string) { + val := ctx.RespHeader().Get("Access-Control-Expose-Headers") + if len(val) != 0 { + ctx.RespHeader().Set("Access-Control-Expose-Headers", fmt.Sprintf("%s, %s", val, strings.Join(names, ", "))) + } else { + ctx.RespHeader().Set("Access-Control-Expose-Headers", strings.Join(names, ", ")) + } +} + +// Written returns true if there are something sent to web browser +func (ctx *Context) Written() bool { + return ctx.Resp.Status() > 0 +} + +// Status writes status code +func (ctx *Context) Status(status int) { + ctx.Resp.WriteHeader(status) +} + +// Write writes data to web browser +func (ctx *Context) Write(bs []byte) (int, error) { + return ctx.Resp.Write(bs) +} + +// RedirectToUser redirect to a differently-named user +func RedirectToUser(ctx *Context, userName string, redirectUserID int64) { + user, err := user_model.GetUserByID(ctx, redirectUserID) + if err != nil { + ctx.ServerError("GetUserByID", err) + return + } + + redirectPath := strings.Replace( + ctx.Req.URL.EscapedPath(), + url.PathEscape(userName), + url.PathEscape(user.Name), + 1, + ) + if ctx.Req.URL.RawQuery != "" { + redirectPath += "?" + ctx.Req.URL.RawQuery + } + ctx.Redirect(path.Join(setting.AppSubURL, redirectPath), http.StatusTemporaryRedirect) +} + +// RedirectToFirst redirects to first not empty URL +func (ctx *Context) RedirectToFirst(location ...string) { + for _, loc := range location { + if len(loc) == 0 { + continue + } + + // Unfortunately browsers consider a redirect Location with preceding "//" and "/\" as meaning redirect to "http(s)://REST_OF_PATH" + // Therefore we should ignore these redirect locations to prevent open redirects + if len(loc) > 1 && loc[0] == '/' && (loc[1] == '/' || loc[1] == '\\') { + continue + } + + u, err := url.Parse(loc) + if err != nil || ((u.Scheme != "" || u.Host != "") && !strings.HasPrefix(strings.ToLower(loc), strings.ToLower(setting.AppURL))) { + continue + } + + ctx.Redirect(loc) + return + } + + ctx.Redirect(setting.AppSubURL + "/") +} + +const tplStatus500 base.TplName = "status/500" + +// HTML calls Context.HTML and renders the template to HTTP response +func (ctx *Context) HTML(status int, name base.TplName) { + log.Debug("Template: %s", name) + + tmplStartTime := time.Now() + if !setting.IsProd { + ctx.Data["TemplateName"] = name + } + ctx.Data["TemplateLoadTimes"] = func() string { + return strconv.FormatInt(time.Since(tmplStartTime).Nanoseconds()/1e6, 10) + "ms" + } + + err := ctx.Render.HTML(ctx.Resp, status, string(name), ctx.Data) + if err == nil { + return + } + + // if rendering fails, show error page + if name != tplStatus500 { + err = fmt.Errorf("failed to render template: %s, error: %s", name, templates.HandleTemplateRenderingError(err)) + ctx.ServerError("Render failed", err) // show the 500 error page + } else { + ctx.PlainText(http.StatusInternalServerError, "Unable to render status/500 page, the template system is broken, or Gitea can't find your template files.") + return + } +} + +// RenderToString renders the template content to a string +func (ctx *Context) RenderToString(name base.TplName, data map[string]interface{}) (string, error) { + var buf strings.Builder + err := ctx.Render.HTML(&buf, http.StatusOK, string(name), data) + return buf.String(), err +} + +// RenderWithErr used for page has form validation but need to prompt error to users. +func (ctx *Context) RenderWithErr(msg string, tpl base.TplName, form interface{}) { + if form != nil { + middleware.AssignForm(form, ctx.Data) + } + ctx.Flash.ErrorMsg = msg + ctx.Data["Flash"] = ctx.Flash + ctx.HTML(http.StatusOK, tpl) +} + +// NotFound displays a 404 (Not Found) page and prints the given error, if any. +func (ctx *Context) NotFound(logMsg string, logErr error) { + ctx.notFoundInternal(logMsg, logErr) +} + +func (ctx *Context) notFoundInternal(logMsg string, logErr error) { + if logErr != nil { + log.Log(2, log.DEBUG, "%s: %v", logMsg, logErr) + if !setting.IsProd { + ctx.Data["ErrorMsg"] = logErr + } + } + + // response simple message if Accept isn't text/html + showHTML := false + for _, part := range ctx.Req.Header["Accept"] { + if strings.Contains(part, "text/html") { + showHTML = true + break + } + } + + if !showHTML { + ctx.plainTextInternal(3, http.StatusNotFound, []byte("Not found.\n")) + return + } + + ctx.Data["IsRepo"] = ctx.Repo.Repository != nil + ctx.Data["Title"] = "Page Not Found" + ctx.HTML(http.StatusNotFound, base.TplName("status/404")) +} + +// ServerError displays a 500 (Internal Server Error) page and prints the given error, if any. +func (ctx *Context) ServerError(logMsg string, logErr error) { + ctx.serverErrorInternal(logMsg, logErr) +} + +func (ctx *Context) serverErrorInternal(logMsg string, logErr error) { + if logErr != nil { + log.ErrorWithSkip(2, "%s: %v", logMsg, logErr) + if _, ok := logErr.(*net.OpError); ok || errors.Is(logErr, &net.OpError{}) { + // This is an error within the underlying connection + // and further rendering will not work so just return + return + } + + // it's safe to show internal error to admin users, and it helps + if !setting.IsProd || (ctx.Doer != nil && ctx.Doer.IsAdmin) { + ctx.Data["ErrorMsg"] = fmt.Sprintf("%s, %s", logMsg, logErr) + } + } + + ctx.Data["Title"] = "Internal Server Error" + ctx.HTML(http.StatusInternalServerError, tplStatus500) +} + +// NotFoundOrServerError use error check function to determine if the error +// is about not found. It responds with 404 status code for not found error, +// or error context description for logging purpose of 500 server error. +func (ctx *Context) NotFoundOrServerError(logMsg string, errCheck func(error) bool, logErr error) { + if errCheck(logErr) { + ctx.notFoundInternal(logMsg, logErr) + return + } + ctx.serverErrorInternal(logMsg, logErr) +} + +// PlainTextBytes renders bytes as plain text +func (ctx *Context) plainTextInternal(skip, status int, bs []byte) { + statusPrefix := status / 100 + if statusPrefix == 4 || statusPrefix == 5 { + log.Log(skip, log.TRACE, "plainTextInternal (status=%d): %s", status, string(bs)) + } + ctx.Resp.Header().Set("Content-Type", "text/plain;charset=utf-8") + ctx.Resp.Header().Set("X-Content-Type-Options", "nosniff") + ctx.Resp.WriteHeader(status) + if _, err := ctx.Resp.Write(bs); err != nil { + log.ErrorWithSkip(skip, "plainTextInternal (status=%d): write bytes failed: %v", status, err) + } +} + +// PlainTextBytes renders bytes as plain text +func (ctx *Context) PlainTextBytes(status int, bs []byte) { + ctx.plainTextInternal(2, status, bs) +} + +// PlainText renders content as plain text +func (ctx *Context) PlainText(status int, text string) { + ctx.plainTextInternal(2, status, []byte(text)) +} + +// RespHeader returns the response header +func (ctx *Context) RespHeader() http.Header { + return ctx.Resp.Header() +} + +// Error returned an error to web browser +func (ctx *Context) Error(status int, contents ...string) { + v := http.StatusText(status) + if len(contents) > 0 { + v = contents[0] + } + http.Error(ctx.Resp, v, status) +} + +// JSON render content as JSON +func (ctx *Context) JSON(status int, content interface{}) { + ctx.Resp.Header().Set("Content-Type", "application/json;charset=utf-8") + ctx.Resp.WriteHeader(status) + if err := json.NewEncoder(ctx.Resp).Encode(content); err != nil { + ctx.ServerError("Render JSON failed", err) + } +} + +// Redirect redirects the request +func (ctx *Context) Redirect(location string, status ...int) { + code := http.StatusSeeOther + if len(status) == 1 { + code = status[0] + } + + if strings.Contains(location, "://") || strings.HasPrefix(location, "//") { + // Some browsers (Safari) have buggy behavior for Cookie + Cache + External Redirection, eg: /my-path => https://other/path + // 1. the first request to "/my-path" contains cookie + // 2. some time later, the request to "/my-path" doesn't contain cookie (caused by Prevent web tracking) + // 3. Gitea's Sessioner doesn't see the session cookie, so it generates a new session id, and returns it to browser + // 4. then the browser accepts the empty session, then the user is logged out + // So in this case, we should remove the session cookie from the response header + removeSessionCookieHeader(ctx.Resp) + } + http.Redirect(ctx.Resp, ctx.Req, location, code) +} diff --git a/modules/context/context_serve.go b/modules/context/context_serve.go new file mode 100644 index 00000000000..5569efbc7eb --- /dev/null +++ b/modules/context/context_serve.go @@ -0,0 +1,23 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package context + +import ( + "io" + "net/http" + + "code.gitea.io/gitea/modules/httplib" +) + +type ServeHeaderOptions httplib.ServeHeaderOptions + +func (ctx *Context) SetServeHeaders(opt *ServeHeaderOptions) { + httplib.ServeSetHeaders(ctx.Resp, (*httplib.ServeHeaderOptions)(opt)) +} + +// ServeContent serves content to http request +func (ctx *Context) ServeContent(r io.ReadSeeker, opts *ServeHeaderOptions) { + httplib.ServeSetHeaders(ctx.Resp, (*httplib.ServeHeaderOptions)(opts)) + http.ServeContent(ctx.Resp, ctx.Req, opts.Filename, opts.LastModified, r) +} diff --git a/modules/context/context_test.go b/modules/context/context_test.go index e1460c1fd70..a6facc97880 100644 --- a/modules/context/context_test.go +++ b/modules/context/context_test.go @@ -7,32 +7,16 @@ import ( "net/http" "testing" + "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" ) -type mockResponseWriter struct { - header http.Header -} - -func (m *mockResponseWriter) Header() http.Header { - return m.header -} - -func (m *mockResponseWriter) Write(bytes []byte) (int, error) { - panic("implement me") -} - -func (m *mockResponseWriter) WriteHeader(statusCode int) { - panic("implement me") -} - func TestRemoveSessionCookieHeader(t *testing.T) { - w := &mockResponseWriter{} - w.header = http.Header{} - w.header.Add("Set-Cookie", (&http.Cookie{Name: setting.SessionConfig.CookieName, Value: "foo"}).String()) - w.header.Add("Set-Cookie", (&http.Cookie{Name: "other", Value: "bar"}).String()) + w := httplib.NewMockResponseWriter() + w.Header().Add("Set-Cookie", (&http.Cookie{Name: setting.SessionConfig.CookieName, Value: "foo"}).String()) + w.Header().Add("Set-Cookie", (&http.Cookie{Name: "other", Value: "bar"}).String()) assert.Len(t, w.Header().Values("Set-Cookie"), 2) removeSessionCookieHeader(w) assert.Len(t, w.Header().Values("Set-Cookie"), 1) diff --git a/modules/context/package.go b/modules/context/package.go index 6c418b31646..fe5bdac19d6 100644 --- a/modules/context/package.go +++ b/modules/context/package.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/templates" + "code.gitea.io/gitea/modules/web/middleware" ) // Package contains owner, access mode and optional the package descriptor @@ -136,7 +137,7 @@ func PackageContexter(ctx gocontext.Context) func(next http.Handler) http.Handle return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { ctx := Context{ Resp: NewResponse(resp), - Data: map[string]interface{}{}, + Data: middleware.GetContextData(req.Context()), Render: rnd, } defer ctx.Close() diff --git a/modules/context/private.go b/modules/context/private.go index 24f50fa4713..f621dd68390 100644 --- a/modules/context/private.go +++ b/modules/context/private.go @@ -11,6 +11,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/process" + "code.gitea.io/gitea/modules/web/middleware" ) // PrivateContext represents a context for private routes @@ -62,7 +63,7 @@ func PrivateContexter() func(http.Handler) http.Handler { ctx := &PrivateContext{ Context: &Context{ Resp: NewResponse(w), - Data: map[string]interface{}{}, + Data: middleware.GetContextData(req.Context()), }, } defer ctx.Close() diff --git a/modules/context/repo.go b/modules/context/repo.go index 6b811054c23..84e07ab4228 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "html" - "io" "net/http" "net/url" "path" @@ -25,37 +24,15 @@ import ( "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" code_indexer "code.gitea.io/gitea/modules/indexer/code" - "code.gitea.io/gitea/modules/issue/template" "code.gitea.io/gitea/modules/log" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" - api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" asymkey_service "code.gitea.io/gitea/services/asymkey" "github.com/editorconfig/editorconfig-core-go/v2" - "gopkg.in/yaml.v3" ) -// IssueTemplateDirCandidates issue templates directory -var IssueTemplateDirCandidates = []string{ - "ISSUE_TEMPLATE", - "issue_template", - ".gitea/ISSUE_TEMPLATE", - ".gitea/issue_template", - ".github/ISSUE_TEMPLATE", - ".github/issue_template", - ".gitlab/ISSUE_TEMPLATE", - ".gitlab/issue_template", -} - -var IssueConfigCandidates = []string{ - ".gitea/ISSUE_TEMPLATE/config", - ".gitea/issue_template/config", - ".github/ISSUE_TEMPLATE/config", - ".github/issue_template/config", -} - // PullRequest contains information to make a pull request type PullRequest struct { BaseRepo *repo_model.Repository @@ -208,7 +185,12 @@ func (r *Repository) GetCommitGraphsCount(ctx context.Context, hidePRRefs bool, if len(branches) == 0 { return git.AllCommitsCount(ctx, r.Repository.RepoPath(), hidePRRefs, files...) } - return git.CommitsCountFiles(ctx, r.Repository.RepoPath(), branches, files) + return git.CommitsCount(ctx, + git.CommitsCountOptions{ + RepoPath: r.Repository.RepoPath(), + Revision: branches, + RelPath: files, + }) }) } @@ -1057,161 +1039,3 @@ func UnitTypes() func(ctx *Context) { ctx.Data["UnitTypeActions"] = unit_model.TypeActions } } - -// IssueTemplatesFromDefaultBranch checks for valid issue templates in the repo's default branch, -func (ctx *Context) IssueTemplatesFromDefaultBranch() []*api.IssueTemplate { - ret, _ := ctx.IssueTemplatesErrorsFromDefaultBranch() - return ret -} - -// IssueTemplatesErrorsFromDefaultBranch checks for issue templates in the repo's default branch, -// returns valid templates and the errors of invalid template files. -func (ctx *Context) IssueTemplatesErrorsFromDefaultBranch() ([]*api.IssueTemplate, map[string]error) { - var issueTemplates []*api.IssueTemplate - - if ctx.Repo.Repository.IsEmpty { - return issueTemplates, nil - } - - if ctx.Repo.Commit == nil { - var err error - ctx.Repo.Commit, err = ctx.Repo.GitRepo.GetBranchCommit(ctx.Repo.Repository.DefaultBranch) - if err != nil { - return issueTemplates, nil - } - } - - invalidFiles := map[string]error{} - for _, dirName := range IssueTemplateDirCandidates { - tree, err := ctx.Repo.Commit.SubTree(dirName) - if err != nil { - log.Debug("get sub tree of %s: %v", dirName, err) - continue - } - entries, err := tree.ListEntries() - if err != nil { - log.Debug("list entries in %s: %v", dirName, err) - return issueTemplates, nil - } - for _, entry := range entries { - if !template.CouldBe(entry.Name()) { - continue - } - fullName := path.Join(dirName, entry.Name()) - if it, err := template.UnmarshalFromEntry(entry, dirName); err != nil { - invalidFiles[fullName] = err - } else { - if !strings.HasPrefix(it.Ref, "refs/") { // Assume that the ref intended is always a branch - for tags users should use refs/tags/ - it.Ref = git.BranchPrefix + it.Ref - } - issueTemplates = append(issueTemplates, it) - } - } - } - return issueTemplates, invalidFiles -} - -func GetDefaultIssueConfig() api.IssueConfig { - return api.IssueConfig{ - BlankIssuesEnabled: true, - ContactLinks: make([]api.IssueConfigContactLink, 0), - } -} - -// GetIssueConfig loads the given issue config file. -// It never returns a nil config. -func (r *Repository) GetIssueConfig(path string, commit *git.Commit) (api.IssueConfig, error) { - if r.GitRepo == nil { - return GetDefaultIssueConfig(), nil - } - - var err error - - treeEntry, err := commit.GetTreeEntryByPath(path) - if err != nil { - return GetDefaultIssueConfig(), err - } - - reader, err := treeEntry.Blob().DataAsync() - if err != nil { - log.Debug("DataAsync: %v", err) - return GetDefaultIssueConfig(), nil - } - - defer reader.Close() - - configContent, err := io.ReadAll(reader) - if err != nil { - return GetDefaultIssueConfig(), err - } - - issueConfig := api.IssueConfig{} - if err := yaml.Unmarshal(configContent, &issueConfig); err != nil { - return GetDefaultIssueConfig(), err - } - - for pos, link := range issueConfig.ContactLinks { - if link.Name == "" { - return GetDefaultIssueConfig(), fmt.Errorf("contact_link at position %d is missing name key", pos+1) - } - - if link.URL == "" { - return GetDefaultIssueConfig(), fmt.Errorf("contact_link at position %d is missing url key", pos+1) - } - - if link.About == "" { - return GetDefaultIssueConfig(), fmt.Errorf("contact_link at position %d is missing about key", pos+1) - } - - _, err = url.ParseRequestURI(link.URL) - if err != nil { - return GetDefaultIssueConfig(), fmt.Errorf("%s is not a valid URL", link.URL) - } - } - - return issueConfig, nil -} - -// IssueConfigFromDefaultBranch returns the issue config for this repo. -// It never returns a nil config. -func (ctx *Context) IssueConfigFromDefaultBranch() (api.IssueConfig, error) { - if ctx.Repo.Repository.IsEmpty { - return GetDefaultIssueConfig(), nil - } - - commit, err := ctx.Repo.GitRepo.GetBranchCommit(ctx.Repo.Repository.DefaultBranch) - if err != nil { - return GetDefaultIssueConfig(), err - } - - for _, configName := range IssueConfigCandidates { - if _, err := commit.GetTreeEntryByPath(configName + ".yaml"); err == nil { - return ctx.Repo.GetIssueConfig(configName+".yaml", commit) - } - - if _, err := commit.GetTreeEntryByPath(configName + ".yml"); err == nil { - return ctx.Repo.GetIssueConfig(configName+".yml", commit) - } - } - - return GetDefaultIssueConfig(), nil -} - -// IsIssueConfig returns if the given path is a issue config file. -func (r *Repository) IsIssueConfig(path string) bool { - for _, configName := range IssueConfigCandidates { - if path == configName+".yaml" || path == configName+".yml" { - return true - } - } - return false -} - -func (ctx *Context) HasIssueTemplatesOrContactLinks() bool { - if len(ctx.IssueTemplatesFromDefaultBranch()) > 0 { - return true - } - - issueConfig, _ := ctx.IssueConfigFromDefaultBranch() - return len(issueConfig.ContactLinks) > 0 -} diff --git a/modules/doctor/doctor.go b/modules/doctor/doctor.go index b23805bc4c9..32eb5938c30 100644 --- a/modules/doctor/doctor.go +++ b/modules/doctor/doctor.go @@ -44,8 +44,7 @@ func (w *wrappedLevelLogger) Log(skip int, level log.Level, format string, v ... } func initDBDisableConsole(ctx context.Context, disableConsole bool) error { - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) setting.LoadDBSetting() setting.InitSQLLog(disableConsole) if err := db.InitEngine(ctx); err != nil { diff --git a/modules/doctor/paths.go b/modules/doctor/paths.go index 1558efc25b9..957152349c2 100644 --- a/modules/doctor/paths.go +++ b/modules/doctor/paths.go @@ -66,8 +66,7 @@ func checkConfigurationFiles(ctx context.Context, logger log.Logger, autofix boo return err } - setting.InitProviderFromExistingFile() - setting.LoadCommonSettings() + setting.Init(&setting.Options{}) configurationFiles := []configurationFile{ {"Configuration File Path", setting.CustomConf, false, true, false}, diff --git a/modules/git/commit.go b/modules/git/commit.go index f28c315cb57..ff654f394d2 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -160,15 +160,29 @@ func AllCommitsCount(ctx context.Context, repoPath string, hidePRRefs bool, file return strconv.ParseInt(strings.TrimSpace(stdout), 10, 64) } -// CommitsCountFiles returns number of total commits of until given revision. -func CommitsCountFiles(ctx context.Context, repoPath string, revision, relpath []string) (int64, error) { +// CommitsCountOptions the options when counting commits +type CommitsCountOptions struct { + RepoPath string + Not string + Revision []string + RelPath []string +} + +// CommitsCount returns number of total commits of until given revision. +func CommitsCount(ctx context.Context, opts CommitsCountOptions) (int64, error) { cmd := NewCommand(ctx, "rev-list", "--count") - cmd.AddDynamicArguments(revision...) - if len(relpath) > 0 { - cmd.AddDashesAndList(relpath...) + + cmd.AddDynamicArguments(opts.Revision...) + + if opts.Not != "" { + cmd.AddOptionValues("--not", opts.Not) } - stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + if len(opts.RelPath) > 0 { + cmd.AddDashesAndList(opts.RelPath...) + } + + stdout, _, err := cmd.RunStdString(&RunOpts{Dir: opts.RepoPath}) if err != nil { return 0, err } @@ -176,14 +190,12 @@ func CommitsCountFiles(ctx context.Context, repoPath string, revision, relpath [ return strconv.ParseInt(strings.TrimSpace(stdout), 10, 64) } -// CommitsCount returns number of total commits of until given revision. -func CommitsCount(ctx context.Context, repoPath string, revision ...string) (int64, error) { - return CommitsCountFiles(ctx, repoPath, revision, []string{}) -} - // CommitsCount returns number of total commits of until current revision. func (c *Commit) CommitsCount() (int64, error) { - return CommitsCount(c.repo.Ctx, c.repo.Path, c.ID.String()) + return CommitsCount(c.repo.Ctx, CommitsCountOptions{ + RepoPath: c.repo.Path, + Revision: []string{c.ID.String()}, + }) } // CommitsByRange returns the specific page commits before current revision, every page's number default by CommitsRangeSize diff --git a/modules/git/commit_test.go b/modules/git/commit_test.go index 1d6fb001832..acf4beb029f 100644 --- a/modules/git/commit_test.go +++ b/modules/git/commit_test.go @@ -14,11 +14,30 @@ import ( func TestCommitsCount(t *testing.T) { bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") - commitsCount, err := CommitsCount(DefaultContext, bareRepo1Path, "8006ff9adbf0cb94da7dad9e537e53817f9fa5c0") + commitsCount, err := CommitsCount(DefaultContext, + CommitsCountOptions{ + RepoPath: bareRepo1Path, + Revision: []string{"8006ff9adbf0cb94da7dad9e537e53817f9fa5c0"}, + }) + assert.NoError(t, err) assert.Equal(t, int64(3), commitsCount) } +func TestCommitsCountWithoutBase(t *testing.T) { + bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") + + commitsCount, err := CommitsCount(DefaultContext, + CommitsCountOptions{ + RepoPath: bareRepo1Path, + Not: "master", + Revision: []string{"branch1"}, + }) + + assert.NoError(t, err) + assert.Equal(t, int64(2), commitsCount) +} + func TestGetFullCommitID(t *testing.T) { bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") diff --git a/modules/git/repo.go b/modules/git/repo.go index 36e0d84dde2..fce99bafa00 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -312,35 +312,28 @@ type DivergeObject struct { Behind int } -func checkDivergence(ctx context.Context, repoPath, baseBranch, targetBranch string) (int, error) { - branches := fmt.Sprintf("%s..%s", baseBranch, targetBranch) - cmd := NewCommand(ctx, "rev-list", "--count").AddDynamicArguments(branches) +// GetDivergingCommits returns the number of commits a targetBranch is ahead or behind a baseBranch +func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch string) (do DivergeObject, err error) { + cmd := NewCommand(ctx, "rev-list", "--count", "--left-right"). + AddDynamicArguments(baseBranch + "..." + targetBranch) stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) if err != nil { - return -1, err + return do, err } - outInteger, errInteger := strconv.Atoi(strings.Trim(stdout, "\n")) - if errInteger != nil { - return -1, errInteger - } - return outInteger, nil -} - -// GetDivergingCommits returns the number of commits a targetBranch is ahead or behind a baseBranch -func GetDivergingCommits(ctx context.Context, repoPath, baseBranch, targetBranch string) (DivergeObject, error) { - // $(git rev-list --count master..feature) commits ahead of master - ahead, errorAhead := checkDivergence(ctx, repoPath, baseBranch, targetBranch) - if errorAhead != nil { - return DivergeObject{}, errorAhead + left, right, found := strings.Cut(strings.Trim(stdout, "\n"), "\t") + if !found { + return do, fmt.Errorf("git rev-list output is missing a tab: %q", stdout) } - // $(git rev-list --count feature..master) commits behind master - behind, errorBehind := checkDivergence(ctx, repoPath, targetBranch, baseBranch) - if errorBehind != nil { - return DivergeObject{}, errorBehind + do.Behind, err = strconv.Atoi(left) + if err != nil { + return do, err } - - return DivergeObject{ahead, behind}, nil + do.Ahead, err = strconv.Atoi(right) + if err != nil { + return do, err + } + return do, nil } // CreateBundle create bundle content to the target path diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index 30a82eb2970..6fc30636291 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -205,12 +205,24 @@ func (repo *Repository) FileChangedBetweenCommits(filename, id1, id2 string) (bo // FileCommitsCount return the number of files at a revision func (repo *Repository) FileCommitsCount(revision, file string) (int64, error) { - return CommitsCountFiles(repo.Ctx, repo.Path, []string{revision}, []string{file}) + return CommitsCount(repo.Ctx, + CommitsCountOptions{ + RepoPath: repo.Path, + Revision: []string{revision}, + RelPath: []string{file}, + }) +} + +type CommitsByFileAndRangeOptions struct { + Revision string + File string + Not string + Page int } // CommitsByFileAndRange return the commits according revision file and the page -func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) ([]*Commit, error) { - skip := (page - 1) * setting.Git.CommitsRangeSize +func (repo *Repository) CommitsByFileAndRange(opts CommitsByFileAndRangeOptions) ([]*Commit, error) { + skip := (opts.Page - 1) * setting.Git.CommitsRangeSize stdoutReader, stdoutWriter := io.Pipe() defer func() { @@ -220,10 +232,15 @@ func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) ( go func() { stderr := strings.Builder{} gitCmd := NewCommand(repo.Ctx, "rev-list"). - AddOptionFormat("--max-count=%d", setting.Git.CommitsRangeSize*page). + AddOptionFormat("--max-count=%d", setting.Git.CommitsRangeSize*opts.Page). AddOptionFormat("--skip=%d", skip) - gitCmd.AddDynamicArguments(revision) - gitCmd.AddDashesAndList(file) + gitCmd.AddDynamicArguments(opts.Revision) + + if opts.Not != "" { + gitCmd.AddOptionValues("--not", opts.Not) + } + + gitCmd.AddDashesAndList(opts.File) err := gitCmd.Run(&RunOpts{ Dir: repo.Path, Stdout: stdoutWriter, @@ -365,11 +382,18 @@ func (repo *Repository) CommitsBetweenIDs(last, before string) ([]*Commit, error // CommitsCountBetween return numbers of commits between two commits func (repo *Repository) CommitsCountBetween(start, end string) (int64, error) { - count, err := CommitsCountFiles(repo.Ctx, repo.Path, []string{start + ".." + end}, []string{}) + count, err := CommitsCount(repo.Ctx, CommitsCountOptions{ + RepoPath: repo.Path, + Revision: []string{start + ".." + end}, + }) + if err != nil && strings.Contains(err.Error(), "no merge base") { // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. // previously it would return the results of git rev-list before last so let's try that... - return CommitsCountFiles(repo.Ctx, repo.Path, []string{start, end}, []string{}) + return CommitsCount(repo.Ctx, CommitsCountOptions{ + RepoPath: repo.Path, + Revision: []string{start, end}, + }) } return count, err diff --git a/modules/git/repo_test.go b/modules/git/repo_test.go index 044b9d40650..9db78153a10 100644 --- a/modules/git/repo_test.go +++ b/modules/git/repo_test.go @@ -4,6 +4,7 @@ package git import ( + "context" "path/filepath" "testing" @@ -29,3 +30,27 @@ func TestRepoIsEmpty(t *testing.T) { assert.NoError(t, err) assert.True(t, isEmpty) } + +func TestRepoGetDivergingCommits(t *testing.T) { + bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") + do, err := GetDivergingCommits(context.Background(), bareRepo1Path, "master", "branch2") + assert.NoError(t, err) + assert.Equal(t, DivergeObject{ + Ahead: 1, + Behind: 5, + }, do) + + do, err = GetDivergingCommits(context.Background(), bareRepo1Path, "master", "master") + assert.NoError(t, err) + assert.Equal(t, DivergeObject{ + Ahead: 0, + Behind: 0, + }, do) + + do, err = GetDivergingCommits(context.Background(), bareRepo1Path, "master", "test") + assert.NoError(t, err) + assert.Equal(t, DivergeObject{ + Ahead: 0, + Behind: 2, + }, do) +} diff --git a/modules/httplib/mock.go b/modules/httplib/mock.go new file mode 100644 index 00000000000..7d284e86fb9 --- /dev/null +++ b/modules/httplib/mock.go @@ -0,0 +1,35 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httplib + +import ( + "bytes" + "net/http" +) + +type MockResponseWriter struct { + header http.Header + + StatusCode int + BodyBuffer bytes.Buffer +} + +func (m *MockResponseWriter) Header() http.Header { + return m.header +} + +func (m *MockResponseWriter) Write(bytes []byte) (int, error) { + if m.StatusCode == 0 { + m.StatusCode = http.StatusOK + } + return m.BodyBuffer.Write(bytes) +} + +func (m *MockResponseWriter) WriteHeader(statusCode int) { + m.StatusCode = statusCode +} + +func NewMockResponseWriter() *MockResponseWriter { + return &MockResponseWriter{header: http.Header{}} +} diff --git a/modules/httplib/httplib.go b/modules/httplib/request.go similarity index 100% rename from modules/httplib/httplib.go rename to modules/httplib/request.go diff --git a/modules/httplib/serve.go b/modules/httplib/serve.go new file mode 100644 index 00000000000..12d68c2d656 --- /dev/null +++ b/modules/httplib/serve.go @@ -0,0 +1,225 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httplib + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + charsetModule "code.gitea.io/gitea/modules/charset" + "code.gitea.io/gitea/modules/httpcache" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/typesniffer" + "code.gitea.io/gitea/modules/util" +) + +type ServeHeaderOptions struct { + ContentType string // defaults to "application/octet-stream" + ContentTypeCharset string + ContentLength *int64 + Disposition string // defaults to "attachment" + Filename string + CacheDuration time.Duration // defaults to 5 minutes + LastModified time.Time +} + +// ServeSetHeaders sets necessary content serve headers +func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) { + header := w.Header() + + contentType := typesniffer.ApplicationOctetStream + if opts.ContentType != "" { + if opts.ContentTypeCharset != "" { + contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset) + } else { + contentType = opts.ContentType + } + } + header.Set("Content-Type", contentType) + header.Set("X-Content-Type-Options", "nosniff") + + if opts.ContentLength != nil { + header.Set("Content-Length", strconv.FormatInt(*opts.ContentLength, 10)) + } + + if opts.Filename != "" { + disposition := opts.Disposition + if disposition == "" { + disposition = "attachment" + } + + backslashEscapedName := strings.ReplaceAll(strings.ReplaceAll(opts.Filename, `\`, `\\`), `"`, `\"`) // \ -> \\, " -> \" + header.Set("Content-Disposition", fmt.Sprintf(`%s; filename="%s"; filename*=UTF-8''%s`, disposition, backslashEscapedName, url.PathEscape(opts.Filename))) + header.Set("Access-Control-Expose-Headers", "Content-Disposition") + } + + duration := opts.CacheDuration + if duration == 0 { + duration = 5 * time.Minute + } + httpcache.SetCacheControlInHeader(header, duration) + + if !opts.LastModified.IsZero() { + header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat)) + } +} + +// ServeData download file from io.Reader +func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath string, mineBuf []byte) { + // do not set "Content-Length", because the length could only be set by callers, and it needs to support range requests + opts := &ServeHeaderOptions{ + Filename: path.Base(filePath), + } + + sniffedType := typesniffer.DetectContentType(mineBuf) + + // the "render" parameter came from year 2016: 638dd24c, it doesn't have clear meaning, so I think it could be removed later + isPlain := sniffedType.IsText() || r.FormValue("render") != "" + + if setting.MimeTypeMap.Enabled { + fileExtension := strings.ToLower(filepath.Ext(filePath)) + opts.ContentType = setting.MimeTypeMap.Map[fileExtension] + } + + if opts.ContentType == "" { + if sniffedType.IsBrowsableBinaryType() { + opts.ContentType = sniffedType.GetMimeType() + } else if isPlain { + opts.ContentType = "text/plain" + } else { + opts.ContentType = typesniffer.ApplicationOctetStream + } + } + + if isPlain { + charset, err := charsetModule.DetectEncoding(mineBuf) + if err != nil { + log.Error("Detect raw file %s charset failed: %v, using by default utf-8", filePath, err) + charset = "utf-8" + } + opts.ContentTypeCharset = strings.ToLower(charset) + } + + isSVG := sniffedType.IsSvgImage() + + // serve types that can present a security risk with CSP + if isSVG { + w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox") + } else if sniffedType.IsPDF() { + // no sandbox attribute for pdf as it breaks rendering in at least safari. this + // should generally be safe as scripts inside PDF can not escape the PDF document + // see https://bugs.chromium.org/p/chromium/issues/detail?id=413851 for more discussion + w.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'") + } + + opts.Disposition = "inline" + if isSVG && !setting.UI.SVG.Enabled { + opts.Disposition = "attachment" + } + + ServeSetHeaders(w, opts) +} + +const mimeDetectionBufferLen = 1024 + +func ServeContentByReader(r *http.Request, w http.ResponseWriter, filePath string, size int64, reader io.Reader) { + buf := make([]byte, mimeDetectionBufferLen) + n, err := util.ReadAtMost(reader, buf) + if err != nil { + http.Error(w, "serve content: unable to pre-read", http.StatusRequestedRangeNotSatisfiable) + return + } + if n >= 0 { + buf = buf[:n] + } + setServeHeadersByFile(r, w, filePath, buf) + + // reset the reader to the beginning + reader = io.MultiReader(bytes.NewReader(buf), reader) + + rangeHeader := r.Header.Get("Range") + + // if no size or no supported range, serve as 200 (complete response) + if size <= 0 || !strings.HasPrefix(rangeHeader, "bytes=") { + if size >= 0 { + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + } + _, _ = io.Copy(w, reader) // just like http.ServeContent, not necessary to handle the error + return + } + + // do our best to support the minimal "Range" request (no support for multiple range: "Range: bytes=0-50, 100-150") + // + // GET /... + // Range: bytes=0-1023 + // + // HTTP/1.1 206 Partial Content + // Content-Range: bytes 0-1023/146515 + // Content-Length: 1024 + + _, rangeParts, _ := strings.Cut(rangeHeader, "=") + rangeBytesStart, rangeBytesEnd, found := strings.Cut(rangeParts, "-") + start, err := strconv.ParseInt(rangeBytesStart, 10, 64) + if start < 0 || start >= size { + err = errors.New("invalid start range") + } + if err != nil { + http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) + return + } + end, err := strconv.ParseInt(rangeBytesEnd, 10, 64) + if rangeBytesEnd == "" && found { + err = nil + end = size - 1 + } + if end >= size { + end = size - 1 + } + if end < start { + err = errors.New("invalid end range") + } + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + partialLength := end - start + 1 + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, size)) + w.Header().Set("Content-Length", strconv.FormatInt(partialLength, 10)) + if _, err = io.CopyN(io.Discard, reader, start); err != nil { + http.Error(w, "serve content: unable to skip", http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusPartialContent) + _, _ = io.CopyN(w, reader, partialLength) // just like http.ServeContent, not necessary to handle the error +} + +func ServeContentByReadSeeker(r *http.Request, w http.ResponseWriter, filePath string, modTime time.Time, reader io.ReadSeeker) { + buf := make([]byte, mimeDetectionBufferLen) + n, err := util.ReadAtMost(reader, buf) + if err != nil { + http.Error(w, "serve content: unable to read", http.StatusInternalServerError) + return + } + if _, err = reader.Seek(0, io.SeekStart); err != nil { + http.Error(w, "serve content: unable to seek", http.StatusInternalServerError) + return + } + if n >= 0 { + buf = buf[:n] + } + setServeHeadersByFile(r, w, filePath, buf) + http.ServeContent(w, r, path.Base(filePath), modTime, reader) +} diff --git a/modules/httplib/serve_test.go b/modules/httplib/serve_test.go new file mode 100644 index 00000000000..0768f1c7139 --- /dev/null +++ b/modules/httplib/serve_test.go @@ -0,0 +1,109 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package httplib + +import ( + "fmt" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestServeContentByReader(t *testing.T) { + data := "0123456789abcdef" + + test := func(t *testing.T, expectedStatusCode int, expectedContent string) { + _, rangeStr, _ := strings.Cut(t.Name(), "_range_") + r := &http.Request{Header: http.Header{}, Form: url.Values{}} + if rangeStr != "" { + r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + } + reader := strings.NewReader(data) + w := NewMockResponseWriter() + ServeContentByReader(r, w, "test", int64(len(data)), reader) + assert.Equal(t, expectedStatusCode, w.StatusCode) + if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { + assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, expectedContent, w.BodyBuffer.String()) + } + } + + t.Run("_range_", func(t *testing.T) { + test(t, http.StatusOK, data) + }) + t.Run("_range_0-", func(t *testing.T) { + test(t, http.StatusPartialContent, data) + }) + t.Run("_range_0-15", func(t *testing.T) { + test(t, http.StatusPartialContent, data) + }) + t.Run("_range_1-", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:]) + }) + t.Run("_range_1-3", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:3+1]) + }) + t.Run("_range_16-", func(t *testing.T) { + test(t, http.StatusRequestedRangeNotSatisfiable, "") + }) + t.Run("_range_1-99999", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:]) + }) +} + +func TestServeContentByReadSeeker(t *testing.T) { + data := "0123456789abcdef" + tmpFile := t.TempDir() + "/test" + err := os.WriteFile(tmpFile, []byte(data), 0o644) + assert.NoError(t, err) + + test := func(t *testing.T, expectedStatusCode int, expectedContent string) { + _, rangeStr, _ := strings.Cut(t.Name(), "_range_") + r := &http.Request{Header: http.Header{}, Form: url.Values{}} + if rangeStr != "" { + r.Header.Set("Range", fmt.Sprintf("bytes=%s", rangeStr)) + } + + seekReader, err := os.OpenFile(tmpFile, os.O_RDONLY, 0o644) + if !assert.NoError(t, err) { + return + } + defer seekReader.Close() + + w := NewMockResponseWriter() + ServeContentByReadSeeker(r, w, "test", time.Time{}, seekReader) + assert.Equal(t, expectedStatusCode, w.StatusCode) + if expectedStatusCode == http.StatusPartialContent || expectedStatusCode == http.StatusOK { + assert.Equal(t, fmt.Sprint(len(expectedContent)), w.Header().Get("Content-Length")) + assert.Equal(t, expectedContent, w.BodyBuffer.String()) + } + } + + t.Run("_range_", func(t *testing.T) { + test(t, http.StatusOK, data) + }) + t.Run("_range_0-", func(t *testing.T) { + test(t, http.StatusPartialContent, data) + }) + t.Run("_range_0-15", func(t *testing.T) { + test(t, http.StatusPartialContent, data) + }) + t.Run("_range_1-", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:]) + }) + t.Run("_range_1-3", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:3+1]) + }) + t.Run("_range_16-", func(t *testing.T) { + test(t, http.StatusRequestedRangeNotSatisfiable, "") + }) + t.Run("_range_1-99999", func(t *testing.T) { + test(t, http.StatusPartialContent, data[1:]) + }) +} diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go index e9085f4107c..5936613e3ad 100644 --- a/modules/indexer/code/bleve.go +++ b/modules/indexer/code/bleve.go @@ -273,10 +273,6 @@ func (b *BleveIndexer) Close() { log.Info("PID: %d Repository Indexer closed", os.Getpid()) } -// SetAvailabilityChangeCallback does nothing -func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) { -} - // Ping does nothing func (b *BleveIndexer) Ping() bool { return true diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go index 68c80967585..60975380091 100644 --- a/modules/indexer/code/elastic_search.go +++ b/modules/indexer/code/elastic_search.go @@ -42,12 +42,11 @@ var _ Indexer = &ElasticSearchIndexer{} // ElasticSearchIndexer implements Indexer interface type ElasticSearchIndexer struct { - client *elastic.Client - indexerAliasName string - available bool - availabilityCallback func(bool) - stopTimer chan struct{} - lock sync.RWMutex + client *elastic.Client + indexerAliasName string + available bool + stopTimer chan struct{} + lock sync.RWMutex } type elasticLogger struct { @@ -198,13 +197,6 @@ func (b *ElasticSearchIndexer) init() (bool, error) { return exists, nil } -// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes -func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) { - b.lock.Lock() - defer b.lock.Unlock() - b.availabilityCallback = callback -} - // Ping checks if elastic is available func (b *ElasticSearchIndexer) Ping() bool { b.lock.RLock() @@ -529,8 +521,4 @@ func (b *ElasticSearchIndexer) setAvailability(available bool) { } b.available = available - if b.availabilityCallback != nil { - // Call the callback from within the lock to ensure that the ordering remains correct - b.availabilityCallback(b.available) - } } diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index 2c493ccf945..a5e40b52c19 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -44,7 +44,6 @@ type SearchResultLanguages struct { // Indexer defines an interface to index and search code contents type Indexer interface { Ping() bool - SetAvailabilityChangeCallback(callback func(bool)) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error Delete(repoID int64) error Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) @@ -81,7 +80,7 @@ type IndexerData struct { RepoID int64 } -var indexerQueue queue.UniqueQueue +var indexerQueue *queue.WorkerPoolQueue[*IndexerData] func index(ctx context.Context, indexer Indexer, repoID int64) error { repo, err := repo_model.GetRepositoryByID(ctx, repoID) @@ -137,37 +136,45 @@ func Init() { // Create the Queue switch setting.Indexer.RepoType { case "bleve", "elasticsearch": - handler := func(data ...queue.Data) []queue.Data { + handler := func(items ...*IndexerData) (unhandled []*IndexerData) { idx, err := indexer.get() if idx == nil || err != nil { log.Error("Codes indexer handler: unable to get indexer!") - return data + return items } - unhandled := make([]queue.Data, 0, len(data)) - for _, datum := range data { - indexerData, ok := datum.(*IndexerData) - if !ok { - log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum) - continue - } + for _, indexerData := range items { log.Trace("IndexerData Process Repo: %d", indexerData.RepoID) + // FIXME: it seems there is a bug in `CatFileBatch` or `nio.Pipe`, which will cause the process to hang forever in rare cases + /* + sync.(*Cond).Wait(cond.go:70) + github.com/djherbis/nio/v3.(*PipeReader).Read(sync.go:106) + bufio.(*Reader).fill(bufio.go:106) + bufio.(*Reader).ReadSlice(bufio.go:372) + bufio.(*Reader).collectFragments(bufio.go:447) + bufio.(*Reader).ReadString(bufio.go:494) + code.gitea.io/gitea/modules/git.ReadBatchLine(batch_reader.go:149) + code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).addUpdate(bleve.go:214) + code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).Index(bleve.go:296) + code.gitea.io/gitea/modules/indexer/code.(*wrappedIndexer).Index(wrapped.go:74) + code.gitea.io/gitea/modules/indexer/code.index(indexer.go:105) + */ if err := index(ctx, indexer, indexerData.RepoID); err != nil { - if !setting.IsInTesting { - log.Error("indexer index error for repo %v: %v", indexerData.RepoID, err) - } - if indexer.Ping() { + if !idx.Ping() { + log.Error("Code indexer handler: indexer is unavailable.") + unhandled = append(unhandled, indexerData) continue } - // Add back to queue - unhandled = append(unhandled, datum) + if !setting.IsInTesting { + log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err) + } } } return unhandled } - indexerQueue = queue.CreateUniqueQueue("code_indexer", handler, &IndexerData{}) + indexerQueue = queue.CreateUniqueQueue("code_indexer", handler) if indexerQueue == nil { log.Fatal("Unable to create codes indexer queue") } @@ -224,18 +231,6 @@ func Init() { indexer.set(rIndexer) - if queue, ok := indexerQueue.(queue.Pausable); ok { - rIndexer.SetAvailabilityChangeCallback(func(available bool) { - if !available { - log.Info("Code index queue paused") - queue.Pause() - } else { - log.Info("Code index queue resumed") - queue.Resume() - } - }) - } - // Start processing the queue go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run) diff --git a/modules/indexer/code/wrapped.go b/modules/indexer/code/wrapped.go index 33ba57a094c..7eed3e85575 100644 --- a/modules/indexer/code/wrapped.go +++ b/modules/indexer/code/wrapped.go @@ -56,16 +56,6 @@ func (w *wrappedIndexer) get() (Indexer, error) { return w.internal, nil } -// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes -func (w *wrappedIndexer) SetAvailabilityChangeCallback(callback func(bool)) { - indexer, err := w.get() - if err != nil { - log.Error("Failed to get indexer: %v", err) - return - } - indexer.SetAvailabilityChangeCallback(callback) -} - // Ping checks if elastic is available func (w *wrappedIndexer) Ping() bool { indexer, err := w.get() diff --git a/modules/indexer/issues/bleve.go b/modules/indexer/issues/bleve.go index e3ef9af5b9a..60d9ef76174 100644 --- a/modules/indexer/issues/bleve.go +++ b/modules/indexer/issues/bleve.go @@ -187,10 +187,6 @@ func (b *BleveIndexer) Init() (bool, error) { return false, err } -// SetAvailabilityChangeCallback does nothing -func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) { -} - // Ping does nothing func (b *BleveIndexer) Ping() bool { return true diff --git a/modules/indexer/issues/db.go b/modules/indexer/issues/db.go index d28b536e024..04c101c3569 100644 --- a/modules/indexer/issues/db.go +++ b/modules/indexer/issues/db.go @@ -18,10 +18,6 @@ func (i *DBIndexer) Init() (bool, error) { return false, nil } -// SetAvailabilityChangeCallback dummy function -func (i *DBIndexer) SetAvailabilityChangeCallback(callback func(bool)) { -} - // Ping checks if database is available func (i *DBIndexer) Ping() bool { return db.GetEngine(db.DefaultContext).Ping() != nil diff --git a/modules/indexer/issues/elastic_search.go b/modules/indexer/issues/elastic_search.go index ee8e3df62ff..fd1dd4b4523 100644 --- a/modules/indexer/issues/elastic_search.go +++ b/modules/indexer/issues/elastic_search.go @@ -22,12 +22,11 @@ var _ Indexer = &ElasticSearchIndexer{} // ElasticSearchIndexer implements Indexer interface type ElasticSearchIndexer struct { - client *elastic.Client - indexerName string - available bool - availabilityCallback func(bool) - stopTimer chan struct{} - lock sync.RWMutex + client *elastic.Client + indexerName string + available bool + stopTimer chan struct{} + lock sync.RWMutex } type elasticLogger struct { @@ -138,13 +137,6 @@ func (b *ElasticSearchIndexer) Init() (bool, error) { return true, nil } -// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes -func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) { - b.lock.Lock() - defer b.lock.Unlock() - b.availabilityCallback = callback -} - // Ping checks if elastic is available func (b *ElasticSearchIndexer) Ping() bool { b.lock.RLock() @@ -305,8 +297,4 @@ func (b *ElasticSearchIndexer) setAvailability(available bool) { } b.available = available - if b.availabilityCallback != nil { - // Call the callback from within the lock to ensure that the ordering remains correct - b.availabilityCallback(b.available) - } } diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 47a8b10794e..e88b1b2befa 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -49,7 +49,6 @@ type SearchResult struct { type Indexer interface { Init() (bool, error) Ping() bool - SetAvailabilityChangeCallback(callback func(bool)) Index(issue []*IndexerData) error Delete(ids ...int64) error Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) @@ -94,7 +93,7 @@ func (h *indexerHolder) get() Indexer { var ( // issueIndexerQueue queue of issue ids to be updated - issueIndexerQueue queue.Queue + issueIndexerQueue *queue.WorkerPoolQueue[*IndexerData] holder = newIndexerHolder() ) @@ -108,62 +107,44 @@ func InitIssueIndexer(syncReindex bool) { // Create the Queue switch setting.Indexer.IssueType { case "bleve", "elasticsearch", "meilisearch": - handler := func(data ...queue.Data) []queue.Data { + handler := func(items ...*IndexerData) (unhandled []*IndexerData) { indexer := holder.get() if indexer == nil { - log.Error("Issue indexer handler: unable to get indexer!") - return data + log.Error("Issue indexer handler: unable to get indexer.") + return items } - - iData := make([]*IndexerData, 0, len(data)) - unhandled := make([]queue.Data, 0, len(data)) - for _, datum := range data { - indexerData, ok := datum.(*IndexerData) - if !ok { - log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum) - continue - } + toIndex := make([]*IndexerData, 0, len(items)) + for _, indexerData := range items { log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete) if indexerData.IsDelete { if err := indexer.Delete(indexerData.IDs...); err != nil { - log.Error("Error whilst deleting from index: %v Error: %v", indexerData.IDs, err) - if indexer.Ping() { - continue + log.Error("Issue indexer handler: failed to from index: %v Error: %v", indexerData.IDs, err) + if !indexer.Ping() { + log.Error("Issue indexer handler: indexer is unavailable when deleting") + unhandled = append(unhandled, indexerData) } - // Add back to queue - unhandled = append(unhandled, datum) } continue } - iData = append(iData, indexerData) + toIndex = append(toIndex, indexerData) } - if len(unhandled) > 0 { - for _, indexerData := range iData { - unhandled = append(unhandled, indexerData) + if err := indexer.Index(toIndex); err != nil { + log.Error("Error whilst indexing: %v Error: %v", toIndex, err) + if !indexer.Ping() { + log.Error("Issue indexer handler: indexer is unavailable when indexing") + unhandled = append(unhandled, toIndex...) } - return unhandled } - if err := indexer.Index(iData); err != nil { - log.Error("Error whilst indexing: %v Error: %v", iData, err) - if indexer.Ping() { - return nil - } - // Add back to queue - for _, indexerData := range iData { - unhandled = append(unhandled, indexerData) - } - return unhandled - } - return nil + return unhandled } - issueIndexerQueue = queue.CreateQueue("issue_indexer", handler, &IndexerData{}) + issueIndexerQueue = queue.CreateSimpleQueue("issue_indexer", handler) if issueIndexerQueue == nil { log.Fatal("Unable to create issue indexer queue") } default: - issueIndexerQueue = &queue.DummyQueue{} + issueIndexerQueue = queue.CreateSimpleQueue[*IndexerData]("issue_indexer", nil) } // Create the Indexer @@ -240,18 +221,6 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType) } - if queue, ok := issueIndexerQueue.(queue.Pausable); ok { - holder.get().SetAvailabilityChangeCallback(func(available bool) { - if !available { - log.Info("Issue index queue paused") - queue.Pause() - } else { - log.Info("Issue index queue resumed") - queue.Resume() - } - }) - } - // Start processing the queue go graceful.GetManager().RunWithShutdownFns(issueIndexerQueue.Run) @@ -285,9 +254,7 @@ func InitIssueIndexer(syncReindex bool) { case <-graceful.GetManager().IsShutdown(): log.Warn("Shutdown occurred before issue index initialisation was complete") case <-time.After(timeout): - if shutdownable, ok := issueIndexerQueue.(queue.Shutdownable); ok { - shutdownable.Terminate() - } + issueIndexerQueue.ShutdownWait(5 * time.Second) log.Fatal("Issue Indexer Initialization timed-out after: %v", timeout) } }() diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index c7c7cea90f0..a2d1794f4b6 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -26,7 +26,7 @@ func TestMain(m *testing.M) { func TestBleveSearchIssues(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - setting.CfgProvider = setting.NewEmptyConfigProvider() + setting.CfgProvider, _ = setting.NewConfigProviderFromData("") tmpIndexerDir := t.TempDir() diff --git a/modules/indexer/issues/meilisearch.go b/modules/indexer/issues/meilisearch.go index 319dc3e30b2..990bc57a05f 100644 --- a/modules/indexer/issues/meilisearch.go +++ b/modules/indexer/issues/meilisearch.go @@ -17,12 +17,11 @@ var _ Indexer = &MeilisearchIndexer{} // MeilisearchIndexer implements Indexer interface type MeilisearchIndexer struct { - client *meilisearch.Client - indexerName string - available bool - availabilityCallback func(bool) - stopTimer chan struct{} - lock sync.RWMutex + client *meilisearch.Client + indexerName string + available bool + stopTimer chan struct{} + lock sync.RWMutex } // MeilisearchIndexer creates a new meilisearch indexer @@ -73,13 +72,6 @@ func (b *MeilisearchIndexer) Init() (bool, error) { return false, b.checkError(err) } -// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes -func (b *MeilisearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) { - b.lock.Lock() - defer b.lock.Unlock() - b.availabilityCallback = callback -} - // Ping checks if meilisearch is available func (b *MeilisearchIndexer) Ping() bool { b.lock.RLock() @@ -178,8 +170,4 @@ func (b *MeilisearchIndexer) setAvailability(available bool) { } b.available = available - if b.availabilityCallback != nil { - // Call the callback from within the lock to ensure that the ordering remains correct - b.availabilityCallback(b.available) - } } diff --git a/modules/indexer/stats/indexer_test.go b/modules/indexer/stats/indexer_test.go index 8d9b4e36d9b..2d9844f8c94 100644 --- a/modules/indexer/stats/indexer_test.go +++ b/modules/indexer/stats/indexer_test.go @@ -28,7 +28,7 @@ func TestMain(m *testing.M) { func TestRepoStatsIndex(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - setting.CfgProvider = setting.NewEmptyConfigProvider() + setting.CfgProvider, _ = setting.NewConfigProviderFromData("") setting.LoadQueueSettings() @@ -41,7 +41,7 @@ func TestRepoStatsIndex(t *testing.T) { err = UpdateRepoIndexer(repo) assert.NoError(t, err) - queue.GetManager().FlushAll(context.Background(), 5*time.Second) + assert.NoError(t, queue.GetManager().FlushAll(context.Background(), 5*time.Second)) status, err := repo_model.GetIndexerStatus(db.DefaultContext, repo, repo_model.RepoIndexerTypeStats) assert.NoError(t, err) diff --git a/modules/indexer/stats/queue.go b/modules/indexer/stats/queue.go index a57338e07df..46438925e4c 100644 --- a/modules/indexer/stats/queue.go +++ b/modules/indexer/stats/queue.go @@ -14,12 +14,11 @@ import ( ) // statsQueue represents a queue to handle repository stats updates -var statsQueue queue.UniqueQueue +var statsQueue *queue.WorkerPoolQueue[int64] // handle passed PR IDs and test the PRs -func handle(data ...queue.Data) []queue.Data { - for _, datum := range data { - opts := datum.(int64) +func handler(items ...int64) []int64 { + for _, opts := range items { if err := indexer.Index(opts); err != nil { if !setting.IsInTesting { log.Error("stats queue indexer.Index(%d) failed: %v", opts, err) @@ -30,7 +29,7 @@ func handle(data ...queue.Data) []queue.Data { } func initStatsQueue() error { - statsQueue = queue.CreateUniqueQueue("repo_stats_update", handle, int64(0)) + statsQueue = queue.CreateUniqueQueue("repo_stats_update", handler) if statsQueue == nil { return fmt.Errorf("Unable to create repo_stats_update Queue") } diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 53fac4ab858..daf8c6cfdda 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -18,9 +18,9 @@ import ( var ( // ErrHashMismatch occurs if the content has does not match OID - ErrHashMismatch = errors.New("Content hash does not match OID") + ErrHashMismatch = errors.New("content hash does not match OID") // ErrSizeMismatch occurs if the content size does not match - ErrSizeMismatch = errors.New("Content size does not match") + ErrSizeMismatch = errors.New("content size does not match") ) // ContentStore provides a simple file system based storage. @@ -105,7 +105,7 @@ func (s *ContentStore) Verify(pointer Pointer) (bool, error) { } // ReadMetaObject will read a git_model.LFSMetaObject and return a reader -func ReadMetaObject(pointer Pointer) (io.ReadCloser, error) { +func ReadMetaObject(pointer Pointer) (io.ReadSeekCloser, error) { contentStore := NewContentStore() return contentStore.Get(pointer) } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index cb1216ec946..5e5e4fecbbb 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -28,8 +28,9 @@ var localMetas = map[string]string{ } func TestMain(m *testing.M) { - setting.InitProviderAllowEmpty() - setting.LoadCommonSettings() + setting.Init(&setting.Options{ + AllowEmpty: true, + }) if err := git.InitSimple(context.Background()); err != nil { log.Fatal("git init failed, err: %v", err) } diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index 0c7650a5ffa..e81869d7a44 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -33,8 +33,9 @@ var localMetas = map[string]string{ } func TestMain(m *testing.M) { - setting.InitProviderAllowEmpty() - setting.LoadCommonSettings() + setting.Init(&setting.Options{ + AllowEmpty: true, + }) if err := git.InitSimple(context.Background()); err != nil { log.Fatal("git init failed, err: %v", err) } diff --git a/modules/mirror/mirror.go b/modules/mirror/mirror.go index 37b4c2ac957..73e591adbae 100644 --- a/modules/mirror/mirror.go +++ b/modules/mirror/mirror.go @@ -10,7 +10,7 @@ import ( "code.gitea.io/gitea/modules/setting" ) -var mirrorQueue queue.UniqueQueue +var mirrorQueue *queue.WorkerPoolQueue[*SyncRequest] // SyncType type of sync request type SyncType int @@ -29,11 +29,11 @@ type SyncRequest struct { } // StartSyncMirrors starts a go routine to sync the mirrors -func StartSyncMirrors(queueHandle func(data ...queue.Data) []queue.Data) { +func StartSyncMirrors(queueHandle func(data ...*SyncRequest) []*SyncRequest) { if !setting.Mirror.Enabled { return } - mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest)) + mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle) go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run) } diff --git a/modules/notification/ui/ui.go b/modules/notification/ui/ui.go index 73ea9227482..a4576b6791e 100644 --- a/modules/notification/ui/ui.go +++ b/modules/notification/ui/ui.go @@ -21,7 +21,7 @@ import ( type ( notificationService struct { base.NullNotifier - issueQueue queue.Queue + issueQueue *queue.WorkerPoolQueue[issueNotificationOpts] } issueNotificationOpts struct { @@ -37,13 +37,12 @@ var _ base.Notifier = ¬ificationService{} // NewNotifier create a new notificationService notifier func NewNotifier() base.Notifier { ns := ¬ificationService{} - ns.issueQueue = queue.CreateQueue("notification-service", ns.handle, issueNotificationOpts{}) + ns.issueQueue = queue.CreateSimpleQueue("notification-service", handler) return ns } -func (ns *notificationService) handle(data ...queue.Data) []queue.Data { - for _, datum := range data { - opts := datum.(issueNotificationOpts) +func handler(items ...issueNotificationOpts) []issueNotificationOpts { + for _, opts := range items { if err := activities_model.CreateOrUpdateIssueNotifications(opts.IssueID, opts.CommentID, opts.NotificationAuthorID, opts.ReceiverID); err != nil { log.Error("Was unable to create issue notification: %v", err) } @@ -52,7 +51,7 @@ func (ns *notificationService) handle(data ...queue.Data) []queue.Data { } func (ns *notificationService) Run() { - graceful.GetManager().RunWithShutdownFns(ns.issueQueue.Run) + go graceful.GetManager().RunWithShutdownFns(ns.issueQueue.Run) } func (ns *notificationService) NotifyCreateIssueComment(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, diff --git a/modules/packages/rpm/metadata.go b/modules/packages/rpm/metadata.go new file mode 100644 index 00000000000..607ea42ea0a --- /dev/null +++ b/modules/packages/rpm/metadata.go @@ -0,0 +1,296 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rpm + +import ( + "fmt" + "io" + "strings" + + "code.gitea.io/gitea/modules/timeutil" + "code.gitea.io/gitea/modules/validation" + + "github.com/sassoftware/go-rpmutils" +) + +const ( + PropertyMetadata = "rpm.metdata" + + SettingKeyPrivate = "rpm.key.private" + SettingKeyPublic = "rpm.key.public" + + RepositoryPackage = "_rpm" + RepositoryVersion = "_repository" +) + +const ( + // Can't use the syscall constants because they are not available for windows build. + sIFMT = 0xf000 + sIFDIR = 0x4000 + sIXUSR = 0x40 + sIXGRP = 0x8 + sIXOTH = 0x1 +) + +// https://rpm-software-management.github.io/rpm/manual/spec.html +// https://refspecs.linuxbase.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic/pkgformat.html + +type Package struct { + Name string + Version string + VersionMetadata *VersionMetadata + FileMetadata *FileMetadata +} + +type VersionMetadata struct { + License string `json:"license,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` +} + +type FileMetadata struct { + Architecture string `json:"architecture,omitempty"` + Epoch string `json:"epoch,omitempty"` + Version string `json:"version,omitempty"` + Release string `json:"release,omitempty"` + Vendor string `json:"vendor,omitempty"` + Group string `json:"group,omitempty"` + Packager string `json:"packager,omitempty"` + SourceRpm string `json:"source_rpm,omitempty"` + BuildHost string `json:"build_host,omitempty"` + BuildTime uint64 `json:"build_time,omitempty"` + FileTime uint64 `json:"file_time,omitempty"` + InstalledSize uint64 `json:"installed_size,omitempty"` + ArchiveSize uint64 `json:"archive_size,omitempty"` + + Provides []*Entry `json:"provide,omitempty"` + Requires []*Entry `json:"require,omitempty"` + Conflicts []*Entry `json:"conflict,omitempty"` + Obsoletes []*Entry `json:"obsolete,omitempty"` + + Files []*File `json:"files,omitempty"` + + Changelogs []*Changelog `json:"changelogs,omitempty"` +} + +type Entry struct { + Name string `json:"name" xml:"name,attr"` + Flags string `json:"flags,omitempty" xml:"flags,attr,omitempty"` + Version string `json:"version,omitempty" xml:"ver,attr,omitempty"` + Epoch string `json:"epoch,omitempty" xml:"epoch,attr,omitempty"` + Release string `json:"release,omitempty" xml:"rel,attr,omitempty"` +} + +type File struct { + Path string `json:"path" xml:",chardata"` + Type string `json:"type,omitempty" xml:"type,attr,omitempty"` + IsExecutable bool `json:"is_executable" xml:"-"` +} + +type Changelog struct { + Author string `json:"author,omitempty" xml:"author,attr"` + Date timeutil.TimeStamp `json:"date,omitempty" xml:"date,attr"` + Text string `json:"text,omitempty" xml:",chardata"` +} + +// ParsePackage parses the RPM package file +func ParsePackage(r io.Reader) (*Package, error) { + rpm, err := rpmutils.ReadRpm(r) + if err != nil { + return nil, err + } + + nevra, err := rpm.Header.GetNEVRA() + if err != nil { + return nil, err + } + + version := fmt.Sprintf("%s-%s", nevra.Version, nevra.Release) + if nevra.Epoch != "" && nevra.Epoch != "0" { + version = fmt.Sprintf("%s-%s", nevra.Epoch, version) + } + + p := &Package{ + Name: nevra.Name, + Version: version, + VersionMetadata: &VersionMetadata{ + Summary: getString(rpm.Header, rpmutils.SUMMARY), + Description: getString(rpm.Header, rpmutils.DESCRIPTION), + License: getString(rpm.Header, rpmutils.LICENSE), + ProjectURL: getString(rpm.Header, rpmutils.URL), + }, + FileMetadata: &FileMetadata{ + Architecture: nevra.Arch, + Epoch: nevra.Epoch, + Version: nevra.Version, + Release: nevra.Release, + Vendor: getString(rpm.Header, rpmutils.VENDOR), + Group: getString(rpm.Header, rpmutils.GROUP), + Packager: getString(rpm.Header, rpmutils.PACKAGER), + SourceRpm: getString(rpm.Header, rpmutils.SOURCERPM), + BuildHost: getString(rpm.Header, rpmutils.BUILDHOST), + BuildTime: getUInt64(rpm.Header, rpmutils.BUILDTIME), + FileTime: getUInt64(rpm.Header, rpmutils.FILEMTIMES), + InstalledSize: getUInt64(rpm.Header, rpmutils.SIZE), + ArchiveSize: getUInt64(rpm.Header, rpmutils.SIG_PAYLOADSIZE), + + Provides: getEntries(rpm.Header, rpmutils.PROVIDENAME, rpmutils.PROVIDEVERSION, rpmutils.PROVIDEFLAGS), + Requires: getEntries(rpm.Header, rpmutils.REQUIRENAME, rpmutils.REQUIREVERSION, rpmutils.REQUIREFLAGS), + Conflicts: getEntries(rpm.Header, 1054 /*rpmutils.CONFLICTNAME*/, 1055 /*rpmutils.CONFLICTVERSION*/, 1053 /*rpmutils.CONFLICTFLAGS*/), // https://github.com/sassoftware/go-rpmutils/pull/24 + Obsoletes: getEntries(rpm.Header, rpmutils.OBSOLETENAME, rpmutils.OBSOLETEVERSION, rpmutils.OBSOLETEFLAGS), + Files: getFiles(rpm.Header), + Changelogs: getChangelogs(rpm.Header), + }, + } + + if !validation.IsValidURL(p.VersionMetadata.ProjectURL) { + p.VersionMetadata.ProjectURL = "" + } + + return p, nil +} + +func getString(h *rpmutils.RpmHeader, tag int) string { + values, err := h.GetStrings(tag) + if err != nil || len(values) < 1 { + return "" + } + return values[0] +} + +func getUInt64(h *rpmutils.RpmHeader, tag int) uint64 { + values, err := h.GetUint64s(tag) + if err != nil || len(values) < 1 { + return 0 + } + return values[0] +} + +func getEntries(h *rpmutils.RpmHeader, namesTag, versionsTag, flagsTag int) []*Entry { + names, err := h.GetStrings(namesTag) + if err != nil || len(names) == 0 { + return nil + } + flags, err := h.GetUint64s(flagsTag) + if err != nil || len(flags) == 0 { + return nil + } + versions, err := h.GetStrings(versionsTag) + if err != nil || len(versions) == 0 { + return nil + } + if len(names) != len(flags) || len(names) != len(versions) { + return nil + } + + entries := make([]*Entry, 0, len(names)) + for i := range names { + e := &Entry{ + Name: names[i], + } + + flags := flags[i] + if (flags&rpmutils.RPMSENSE_GREATER) != 0 && (flags&rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "GE" + } else if (flags&rpmutils.RPMSENSE_LESS) != 0 && (flags&rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "LE" + } else if (flags & rpmutils.RPMSENSE_GREATER) != 0 { + e.Flags = "GT" + } else if (flags & rpmutils.RPMSENSE_LESS) != 0 { + e.Flags = "LT" + } else if (flags & rpmutils.RPMSENSE_EQUAL) != 0 { + e.Flags = "EQ" + } + + version := versions[i] + if version != "" { + parts := strings.Split(version, "-") + + versionParts := strings.Split(parts[0], ":") + if len(versionParts) == 2 { + e.Version = versionParts[1] + e.Epoch = versionParts[0] + } else { + e.Version = versionParts[0] + e.Epoch = "0" + } + + if len(parts) > 1 { + e.Release = parts[1] + } + } + + entries = append(entries, e) + } + return entries +} + +func getFiles(h *rpmutils.RpmHeader) []*File { + baseNames, _ := h.GetStrings(rpmutils.BASENAMES) + dirNames, _ := h.GetStrings(rpmutils.DIRNAMES) + dirIndexes, _ := h.GetUint32s(rpmutils.DIRINDEXES) + fileFlags, _ := h.GetUint32s(rpmutils.FILEFLAGS) + fileModes, _ := h.GetUint32s(rpmutils.FILEMODES) + + files := make([]*File, 0, len(baseNames)) + for i := range baseNames { + if len(dirIndexes) <= i { + continue + } + dirIndex := dirIndexes[i] + if len(dirNames) <= int(dirIndex) { + continue + } + + var fileType string + var isExecutable bool + if i < len(fileFlags) && (fileFlags[i]&rpmutils.RPMFILE_GHOST) != 0 { + fileType = "ghost" + } else if i < len(fileModes) { + if (fileModes[i] & sIFMT) == sIFDIR { + fileType = "dir" + } else { + mode := fileModes[i] & ^uint32(sIFMT) + isExecutable = (mode&sIXUSR) != 0 || (mode&sIXGRP) != 0 || (mode&sIXOTH) != 0 + } + } + + files = append(files, &File{ + Path: dirNames[dirIndex] + baseNames[i], + Type: fileType, + IsExecutable: isExecutable, + }) + } + + return files +} + +func getChangelogs(h *rpmutils.RpmHeader) []*Changelog { + texts, err := h.GetStrings(rpmutils.CHANGELOGTEXT) + if err != nil || len(texts) == 0 { + return nil + } + authors, err := h.GetStrings(rpmutils.CHANGELOGNAME) + if err != nil || len(authors) == 0 { + return nil + } + times, err := h.GetUint32s(rpmutils.CHANGELOGTIME) + if err != nil || len(times) == 0 { + return nil + } + if len(texts) != len(authors) || len(texts) != len(times) { + return nil + } + + changelogs := make([]*Changelog, 0, len(texts)) + for i := range texts { + changelogs = append(changelogs, &Changelog{ + Author: authors[i], + Date: timeutil.TimeStamp(times[i]), + Text: texts[i], + }) + } + return changelogs +} diff --git a/modules/packages/rpm/metadata_test.go b/modules/packages/rpm/metadata_test.go new file mode 100644 index 00000000000..bb538ef9d06 --- /dev/null +++ b/modules/packages/rpm/metadata_test.go @@ -0,0 +1,163 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package rpm + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParsePackage(t *testing.T) { + base64RpmPackageContent := `H4sICFayB2QCAGdpdGVhLXRlc3QtMS4wLjItMS14ODZfNjQucnBtAO2YV4gTQRjHJzl7wbNhhxVF +VNwk2zd2PdvZ9Sxnd3Z3NllNsmF3o6congVFsWFHRWwIImIXfRER0QcRfPBJEXvvBQvWSfZTT0VQ +8TF/MuU33zcz3+zOJGEe73lyuQBRBWKWRzDrEddjuVAkxLMc+lsFUOWfm5bvvReAalWECg/TsivU +dyKa0U61aVnl6wj0Uxe4nc8F92hZiaYE8CO/P0r7/Quegr0c7M/AvoCaGZEIWNGUqMHrhhGROIUT +Zc7gOAOraoQzCNZ0WdU0HpEI5jiB4zlek3gT85wqCBomhomxoGCs8wImWMImbxqKgXVNUKKaqShR +STKVKK9glFUNcf2g+/t27xs16v5x/eyOKftVGlIhyiuvvPLKK6+88sorr7zyyiuvvPKCO5HPnz+v +pGVhhXsTsFVeSstuWR9anwU+Bk3Vch5wTwL3JkHg+8C1gR8A169wj1KdpobAj4HbAT+Be5VewE+h +fz/g52AvBX4N9vHAb4AnA7+F8ePAH8BuA38ELgf+BLzQ50oIeBlw0OdAOXAlP57AGuCsbwGtbgCu +DrwRuAb4bwau6T/PwFbgWsDXgWuD/y3gOmC/B1wI/Bi4AcT3Arih3z9YCNzI9w9m/YKUG4Nd9N9z +pSZgHwrcFPgccFt//OADGE+F/q+Ao+D/FrijzwV1gbv4/QvaAHcFDgF3B5aB+wB3Be7rz1dQCtwP +eDxwMcw3GbgU7AasdwzYE8DjwT4L/CeAvRx4IvBCYA3iWQds+FzpDjABfghsAj8BTgA/A/b8+StX +A84A1wKe5s9fuRB4JpzHZv55rL8a/Dv49vpn/PErR4BvQX8Z+Db4l2W5CH2/f0W5+1fEoeFDBzFp +rE/FMcK4mWQSOzN+aDOIqztW2rPsFKIyqh7sQERR42RVMSKihnzVHlQ8Ag0YLBYNEIajkhmuR5Io +7nlpt2M4nJs0ZNkoYaUyZahMlSfJImr1n1WjFVNCPCaTZgYNGdGL8YN2mX8WHfA/C7ViHJK0pxHG +SrkeTiSI4T+7ubf85yrzRCQRQ5EVxVAjvIBVRY/KRFAVReIkhfARSddNSceayQkGliIKb0q8RAxJ +5QWNVxHIsW3Pz369bw+5jh5y0klE9Znqm0dF57b0HbGy2A5lVUBTZZrqZjdUjYoprFmpsBtHP5d0 ++ISltS2yk2mHuC4x+lgJMhgnidvuqy3b0suK0bm+tw3FMxI2zjm7/fA0MtQhplX2s7nYLZ2ZC0yg +CxJZDokhORTJlrlcCvG5OieGBERlVCs7CfuS6WzQ/T2j+9f92BWxTFEcp2IkYccYGp2LYySEfreq +irue4WRF5XkpKovw2wgpq2rZBI8bQZkzxEkiYaNwxnXCCVvHidzIiB3CM2yMYdNWmjDsaLovaE4c +x3a6mLaTxB7rEj3jWN4M2p7uwPaa1GfI8BHFfcZMKhkycnhR7y781/a+A4t7FpWWTupRUtKbegwZ +XMKwJinTSe70uhRcj55qNu3YHtE922Fdz7FTMTq9Q3TbMdiYrrPudMvT44S6u2miu138eC0tTN9D +2CFGHHtQsHHsGCRFDFbXuT9wx6mUTZfseydlkWZeJkW6xOgYjqXT+LA7I6XHaUx2xmUzqelWymA9 +rCXI9+D1BHbjsITssqhBNysw0tOWjcpmIh6+aViYPfftw8ZSGfRVPUqKiosZj5R5qGmk/8AjjRbZ +d8b3vvngdPHx3HvMeCarIk7VVSwbgoZVkceEVyOmyUmGxBGNYDVKSFSOGlIkGqWnUZFkiY/wsmhK +Mu0UFYgZ/bYnuvn/vz4wtCz8qMwsHUvP0PX3tbYFUctAPdrY6tiiDtcCddDECahx7SuVNP5dpmb5 +9tMDyaXb7OAlk5acuPn57ss9mw6Wym0m1Fq2cej7tUt2LL4/b8enXU2fndk+fvv57ndnt55/cQob +7tpp/pEjDS7cGPZ6BY430+7danDq6f42Nw49b9F7zp6BiKpJb9s5P0AYN2+L159cnrur636rx+v1 +7ae1K28QbMMcqI8CqwIrgwg9nTOp8Oj9q81plUY7ZuwXN8Vvs8wbAAA=` + rpmPackageContent, err := base64.StdEncoding.DecodeString(base64RpmPackageContent) + assert.NoError(t, err) + + zr, err := gzip.NewReader(bytes.NewReader(rpmPackageContent)) + assert.NoError(t, err) + + p, err := ParsePackage(zr) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, "gitea-test", p.Name) + assert.Equal(t, "1.0.2-1", p.Version) + assert.NotNil(t, p.VersionMetadata) + assert.NotNil(t, p.FileMetadata) + + assert.Equal(t, "MIT", p.VersionMetadata.License) + assert.Equal(t, "https://gitea.io", p.VersionMetadata.ProjectURL) + assert.Equal(t, "RPM package summary", p.VersionMetadata.Summary) + assert.Equal(t, "RPM package description", p.VersionMetadata.Description) + + assert.Equal(t, "x86_64", p.FileMetadata.Architecture) + assert.Equal(t, "0", p.FileMetadata.Epoch) + assert.Equal(t, "1.0.2", p.FileMetadata.Version) + assert.Equal(t, "1", p.FileMetadata.Release) + assert.Empty(t, p.FileMetadata.Vendor) + assert.Equal(t, "KN4CK3R", p.FileMetadata.Packager) + assert.Equal(t, "gitea-test-1.0.2-1.src.rpm", p.FileMetadata.SourceRpm) + assert.Equal(t, "e44b1687d04b", p.FileMetadata.BuildHost) + assert.EqualValues(t, 1678225964, p.FileMetadata.BuildTime) + assert.EqualValues(t, 1678225964, p.FileMetadata.FileTime) + assert.EqualValues(t, 13, p.FileMetadata.InstalledSize) + assert.EqualValues(t, 272, p.FileMetadata.ArchiveSize) + assert.Empty(t, p.FileMetadata.Conflicts) + assert.Empty(t, p.FileMetadata.Obsoletes) + + assert.ElementsMatch( + t, + []*Entry{ + { + Name: "gitea-test", + Flags: "EQ", + Version: "1.0.2", + Epoch: "0", + Release: "1", + }, + { + Name: "gitea-test(x86-64)", + Flags: "EQ", + Version: "1.0.2", + Epoch: "0", + Release: "1", + }, + }, + p.FileMetadata.Provides, + ) + assert.ElementsMatch( + t, + []*Entry{ + { + Name: "/bin/sh", + }, + { + Name: "/bin/sh", + }, + { + Name: "/bin/sh", + }, + { + Name: "rpmlib(CompressedFileNames)", + Flags: "LE", + Version: "3.0.4", + Epoch: "0", + Release: "1", + }, + { + Name: "rpmlib(FileDigests)", + Flags: "LE", + Version: "4.6.0", + Epoch: "0", + Release: "1", + }, + { + Name: "rpmlib(PayloadFilesHavePrefix)", + Flags: "LE", + Version: "4.0", + Epoch: "0", + Release: "1", + }, + { + Name: "rpmlib(PayloadIsXz)", + Flags: "LE", + Version: "5.2", + Epoch: "0", + Release: "1", + }, + }, + p.FileMetadata.Requires, + ) + assert.ElementsMatch( + t, + []*File{ + { + Path: "/usr/local/bin/hello", + IsExecutable: true, + }, + }, + p.FileMetadata.Files, + ) + assert.ElementsMatch( + t, + []*Changelog{ + { + Author: "KN4CK3R ", + Date: 1678276800, + Text: "- Changelog message.", + }, + }, + p.FileMetadata.Changelogs, + ) +} diff --git a/modules/queue/backoff.go b/modules/queue/backoff.go new file mode 100644 index 00000000000..cda7233567d --- /dev/null +++ b/modules/queue/backoff.go @@ -0,0 +1,63 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "time" +) + +const ( + backoffBegin = 50 * time.Millisecond + backoffUpper = 2 * time.Second +) + +type ( + backoffFuncRetErr[T any] func() (retry bool, ret T, err error) + backoffFuncErr func() (retry bool, err error) +) + +func backoffRetErr[T any](ctx context.Context, begin, upper time.Duration, end <-chan time.Time, fn backoffFuncRetErr[T]) (ret T, err error) { + d := begin + for { + // check whether the context has been cancelled or has reached the deadline, return early + select { + case <-ctx.Done(): + return ret, ctx.Err() + case <-end: + return ret, context.DeadlineExceeded + default: + } + + // call the target function + retry, ret, err := fn() + if err != nil { + return ret, err + } + if !retry { + return ret, nil + } + + // wait for a while before retrying, and also respect the context & deadline + select { + case <-ctx.Done(): + return ret, ctx.Err() + case <-time.After(d): + d *= 2 + if d > upper { + d = upper + } + case <-end: + return ret, context.DeadlineExceeded + } + } +} + +func backoffErr(ctx context.Context, begin, upper time.Duration, end <-chan time.Time, fn backoffFuncErr) error { + _, err := backoffRetErr(ctx, begin, upper, end, func() (retry bool, ret any, err error) { + retry, err = fn() + return retry, nil, err + }) + return err +} diff --git a/modules/queue/base.go b/modules/queue/base.go new file mode 100644 index 00000000000..102e79e5416 --- /dev/null +++ b/modules/queue/base.go @@ -0,0 +1,42 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "time" +) + +var pushBlockTime = 5 * time.Second + +type baseQueue interface { + PushItem(ctx context.Context, data []byte) error + PopItem(ctx context.Context) ([]byte, error) + HasItem(ctx context.Context, data []byte) (bool, error) + Len(ctx context.Context) (int, error) + Close() error + RemoveAll(ctx context.Context) error +} + +func popItemByChan(ctx context.Context, popItemFn func(ctx context.Context) ([]byte, error)) (chanItem chan []byte, chanErr chan error) { + chanItem = make(chan []byte) + chanErr = make(chan error) + go func() { + for { + it, err := popItemFn(ctx) + if err != nil { + close(chanItem) + chanErr <- err + return + } + if it == nil { + close(chanItem) + close(chanErr) + return + } + chanItem <- it + } + }() + return chanItem, chanErr +} diff --git a/modules/queue/base_channel.go b/modules/queue/base_channel.go new file mode 100644 index 00000000000..d03c72bdaef --- /dev/null +++ b/modules/queue/base_channel.go @@ -0,0 +1,131 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "errors" + "sync" + "time" + + "code.gitea.io/gitea/modules/container" +) + +var errChannelClosed = errors.New("channel is closed") + +type baseChannel struct { + c chan []byte + set container.Set[string] + mu sync.Mutex + + isUnique bool +} + +var _ baseQueue = (*baseChannel)(nil) + +func newBaseChannelGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { + q := &baseChannel{c: make(chan []byte, cfg.Length), isUnique: unique} + if unique { + q.set = container.Set[string]{} + } + return q, nil +} + +func newBaseChannelSimple(cfg *BaseConfig) (baseQueue, error) { + return newBaseChannelGeneric(cfg, false) +} + +func newBaseChannelUnique(cfg *BaseConfig) (baseQueue, error) { + return newBaseChannelGeneric(cfg, true) +} + +func (q *baseChannel) PushItem(ctx context.Context, data []byte) error { + if q.c == nil { + return errChannelClosed + } + + if q.isUnique { + q.mu.Lock() + has := q.set.Contains(string(data)) + q.mu.Unlock() + if has { + return ErrAlreadyInQueue + } + } + + select { + case q.c <- data: + if q.isUnique { + q.mu.Lock() + q.set.Add(string(data)) + q.mu.Unlock() + } + return nil + case <-time.After(pushBlockTime): + return context.DeadlineExceeded + case <-ctx.Done(): + return ctx.Err() + } +} + +func (q *baseChannel) PopItem(ctx context.Context) ([]byte, error) { + select { + case data, ok := <-q.c: + if !ok { + return nil, errChannelClosed + } + q.mu.Lock() + q.set.Remove(string(data)) + q.mu.Unlock() + return data, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (q *baseChannel) HasItem(ctx context.Context, data []byte) (bool, error) { + q.mu.Lock() + defer q.mu.Unlock() + if !q.isUnique { + return false, nil + } + return q.set.Contains(string(data)), nil +} + +func (q *baseChannel) Len(ctx context.Context) (int, error) { + q.mu.Lock() + defer q.mu.Unlock() + + if q.c == nil { + return 0, errChannelClosed + } + + return len(q.c), nil +} + +func (q *baseChannel) Close() error { + q.mu.Lock() + defer q.mu.Unlock() + + close(q.c) + if q.isUnique { + q.set = container.Set[string]{} + } + + return nil +} + +func (q *baseChannel) RemoveAll(ctx context.Context) error { + q.mu.Lock() + defer q.mu.Unlock() + + for q.c != nil && len(q.c) > 0 { + <-q.c + } + + if q.isUnique { + q.set = container.Set[string]{} + } + return nil +} diff --git a/modules/queue/base_channel_test.go b/modules/queue/base_channel_test.go new file mode 100644 index 00000000000..5d0a2ed0a7c --- /dev/null +++ b/modules/queue/base_channel_test.go @@ -0,0 +1,11 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import "testing" + +func TestBaseChannel(t *testing.T) { + testQueueBasic(t, newBaseChannelSimple, &BaseConfig{ManagedName: "baseChannel", Length: 10}, false) + testQueueBasic(t, newBaseChannelUnique, &BaseConfig{ManagedName: "baseChannel", Length: 10}, true) +} diff --git a/modules/queue/base_dummy.go b/modules/queue/base_dummy.go new file mode 100644 index 00000000000..7503568a09f --- /dev/null +++ b/modules/queue/base_dummy.go @@ -0,0 +1,38 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import "context" + +type baseDummy struct{} + +var _ baseQueue = (*baseDummy)(nil) + +func newBaseDummy(cfg *BaseConfig, unique bool) (baseQueue, error) { + return &baseDummy{}, nil +} + +func (q *baseDummy) PushItem(ctx context.Context, data []byte) error { + return nil +} + +func (q *baseDummy) PopItem(ctx context.Context) ([]byte, error) { + return nil, nil +} + +func (q *baseDummy) Len(ctx context.Context) (int, error) { + return 0, nil +} + +func (q *baseDummy) HasItem(ctx context.Context, data []byte) (bool, error) { + return false, nil +} + +func (q *baseDummy) Close() error { + return nil +} + +func (q *baseDummy) RemoveAll(ctx context.Context) error { + return nil +} diff --git a/modules/queue/base_levelqueue.go b/modules/queue/base_levelqueue.go new file mode 100644 index 00000000000..afde5021169 --- /dev/null +++ b/modules/queue/base_levelqueue.go @@ -0,0 +1,72 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + + "code.gitea.io/gitea/modules/nosql" + + "gitea.com/lunny/levelqueue" +) + +type baseLevelQueue struct { + internal *levelqueue.Queue + conn string + cfg *BaseConfig +} + +var _ baseQueue = (*baseLevelQueue)(nil) + +func newBaseLevelQueueGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { + if unique { + return newBaseLevelQueueUnique(cfg) + } + return newBaseLevelQueueSimple(cfg) +} + +func newBaseLevelQueueSimple(cfg *BaseConfig) (baseQueue, error) { + conn, db, err := prepareLevelDB(cfg) + if err != nil { + return nil, err + } + q := &baseLevelQueue{conn: conn, cfg: cfg} + q.internal, err = levelqueue.NewQueue(db, []byte(cfg.QueueFullName), false) + if err != nil { + return nil, err + } + + return q, nil +} + +func (q *baseLevelQueue) PushItem(ctx context.Context, data []byte) error { + return baseLevelQueueCommon(q.cfg, q.internal, nil).PushItem(ctx, data) +} + +func (q *baseLevelQueue) PopItem(ctx context.Context) ([]byte, error) { + return baseLevelQueueCommon(q.cfg, q.internal, nil).PopItem(ctx) +} + +func (q *baseLevelQueue) HasItem(ctx context.Context, data []byte) (bool, error) { + return false, nil +} + +func (q *baseLevelQueue) Len(ctx context.Context) (int, error) { + return int(q.internal.Len()), nil +} + +func (q *baseLevelQueue) Close() error { + err := q.internal.Close() + _ = nosql.GetManager().CloseLevelDB(q.conn) + return err +} + +func (q *baseLevelQueue) RemoveAll(ctx context.Context) error { + for q.internal.Len() > 0 { + if _, err := q.internal.LPop(); err != nil { + return err + } + } + return nil +} diff --git a/modules/queue/base_levelqueue_common.go b/modules/queue/base_levelqueue_common.go new file mode 100644 index 00000000000..409a965517e --- /dev/null +++ b/modules/queue/base_levelqueue_common.go @@ -0,0 +1,92 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/nosql" + + "gitea.com/lunny/levelqueue" + "github.com/syndtr/goleveldb/leveldb" +) + +type baseLevelQueuePushPoper interface { + RPush(data []byte) error + LPop() ([]byte, error) + Len() int64 +} + +type baseLevelQueueCommonImpl struct { + length int + internal baseLevelQueuePushPoper + mu *sync.Mutex +} + +func (q *baseLevelQueueCommonImpl) PushItem(ctx context.Context, data []byte) error { + return backoffErr(ctx, backoffBegin, backoffUpper, time.After(pushBlockTime), func() (retry bool, err error) { + if q.mu != nil { + q.mu.Lock() + defer q.mu.Unlock() + } + + cnt := int(q.internal.Len()) + if cnt >= q.length { + return true, nil + } + retry, err = false, q.internal.RPush(data) + if err == levelqueue.ErrAlreadyInQueue { + err = ErrAlreadyInQueue + } + return retry, err + }) +} + +func (q *baseLevelQueueCommonImpl) PopItem(ctx context.Context) ([]byte, error) { + return backoffRetErr(ctx, backoffBegin, backoffUpper, infiniteTimerC, func() (retry bool, data []byte, err error) { + if q.mu != nil { + q.mu.Lock() + defer q.mu.Unlock() + } + + data, err = q.internal.LPop() + if err == levelqueue.ErrNotFound { + return true, nil, nil + } + if err != nil { + return false, nil, err + } + return false, data, nil + }) +} + +func baseLevelQueueCommon(cfg *BaseConfig, internal baseLevelQueuePushPoper, mu *sync.Mutex) *baseLevelQueueCommonImpl { + return &baseLevelQueueCommonImpl{length: cfg.Length, internal: internal} +} + +func prepareLevelDB(cfg *BaseConfig) (conn string, db *leveldb.DB, err error) { + if cfg.ConnStr == "" { // use data dir as conn str + if !filepath.IsAbs(cfg.DataFullDir) { + return "", nil, fmt.Errorf("invalid leveldb data dir (not absolute): %q", cfg.DataFullDir) + } + conn = cfg.DataFullDir + } else { + if !strings.HasPrefix(cfg.ConnStr, "leveldb://") { + return "", nil, fmt.Errorf("invalid leveldb connection string: %q", cfg.ConnStr) + } + conn = cfg.ConnStr + } + for i := 0; i < 10; i++ { + if db, err = nosql.GetManager().GetLevelDB(conn); err == nil { + break + } + time.Sleep(1 * time.Second) + } + return conn, db, err +} diff --git a/modules/queue/base_levelqueue_test.go b/modules/queue/base_levelqueue_test.go new file mode 100644 index 00000000000..712a0892cd4 --- /dev/null +++ b/modules/queue/base_levelqueue_test.go @@ -0,0 +1,23 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestBaseLevelDB(t *testing.T) { + _, err := newBaseLevelQueueGeneric(&BaseConfig{ConnStr: "redis://"}, false) + assert.ErrorContains(t, err, "invalid leveldb connection string") + + _, err = newBaseLevelQueueGeneric(&BaseConfig{DataFullDir: "relative"}, false) + assert.ErrorContains(t, err, "invalid leveldb data dir") + + testQueueBasic(t, newBaseLevelQueueSimple, toBaseConfig("baseLevelQueue", setting.QueueSettings{Datadir: t.TempDir() + "/queue-test", Length: 10}), false) + testQueueBasic(t, newBaseLevelQueueUnique, toBaseConfig("baseLevelQueueUnique", setting.QueueSettings{ConnStr: "leveldb://" + t.TempDir() + "/queue-test", Length: 10}), true) +} diff --git a/modules/queue/base_levelqueue_unique.go b/modules/queue/base_levelqueue_unique.go new file mode 100644 index 00000000000..1acd504e327 --- /dev/null +++ b/modules/queue/base_levelqueue_unique.go @@ -0,0 +1,96 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "sync" + "unsafe" + + "code.gitea.io/gitea/modules/nosql" + + "gitea.com/lunny/levelqueue" + "github.com/syndtr/goleveldb/leveldb" +) + +type baseLevelQueueUnique struct { + internal *levelqueue.UniqueQueue + conn string + cfg *BaseConfig + + mu sync.Mutex // the levelqueue.UniqueQueue is not thread-safe, there is no mutex protecting the underlying queue&set together +} + +var _ baseQueue = (*baseLevelQueueUnique)(nil) + +func newBaseLevelQueueUnique(cfg *BaseConfig) (baseQueue, error) { + conn, db, err := prepareLevelDB(cfg) + if err != nil { + return nil, err + } + q := &baseLevelQueueUnique{conn: conn, cfg: cfg} + q.internal, err = levelqueue.NewUniqueQueue(db, []byte(cfg.QueueFullName), []byte(cfg.SetFullName), false) + if err != nil { + return nil, err + } + + return q, nil +} + +func (q *baseLevelQueueUnique) PushItem(ctx context.Context, data []byte) error { + return baseLevelQueueCommon(q.cfg, q.internal, &q.mu).PushItem(ctx, data) +} + +func (q *baseLevelQueueUnique) PopItem(ctx context.Context) ([]byte, error) { + return baseLevelQueueCommon(q.cfg, q.internal, &q.mu).PopItem(ctx) +} + +func (q *baseLevelQueueUnique) HasItem(ctx context.Context, data []byte) (bool, error) { + q.mu.Lock() + defer q.mu.Unlock() + return q.internal.Has(data) +} + +func (q *baseLevelQueueUnique) Len(ctx context.Context) (int, error) { + q.mu.Lock() + defer q.mu.Unlock() + return int(q.internal.Len()), nil +} + +func (q *baseLevelQueueUnique) Close() error { + q.mu.Lock() + defer q.mu.Unlock() + err := q.internal.Close() + _ = nosql.GetManager().CloseLevelDB(q.conn) + return err +} + +func (q *baseLevelQueueUnique) RemoveAll(ctx context.Context) error { + q.mu.Lock() + defer q.mu.Unlock() + + type levelUniqueQueue struct { + q *levelqueue.Queue + set *levelqueue.Set + db *leveldb.DB + } + lq := (*levelUniqueQueue)(unsafe.Pointer(q.internal)) + + for lq.q.Len() > 0 { + if _, err := lq.q.LPop(); err != nil { + return err + } + } + + // the "set" must be cleared after the "list" because there is no transaction. + // it's better to have duplicate items than losing items. + members, err := lq.set.Members() + if err != nil { + return err // seriously corrupted + } + for _, v := range members { + _, _ = lq.set.Remove(v) + } + return nil +} diff --git a/modules/queue/base_redis.go b/modules/queue/base_redis.go new file mode 100644 index 00000000000..a1e234943d2 --- /dev/null +++ b/modules/queue/base_redis.go @@ -0,0 +1,138 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "sync" + "time" + + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/nosql" + + "github.com/redis/go-redis/v9" +) + +type baseRedis struct { + client redis.UniversalClient + isUnique bool + cfg *BaseConfig + + mu sync.Mutex // the old implementation is not thread-safe, the queue operation and set operation should be protected together +} + +var _ baseQueue = (*baseRedis)(nil) + +func newBaseRedisGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { + client := nosql.GetManager().GetRedisClient(cfg.ConnStr) + + var err error + for i := 0; i < 10; i++ { + err = client.Ping(graceful.GetManager().ShutdownContext()).Err() + if err == nil { + break + } + log.Warn("Redis is not ready, waiting for 1 second to retry: %v", err) + time.Sleep(time.Second) + } + if err != nil { + return nil, err + } + + return &baseRedis{cfg: cfg, client: client, isUnique: unique}, nil +} + +func newBaseRedisSimple(cfg *BaseConfig) (baseQueue, error) { + return newBaseRedisGeneric(cfg, false) +} + +func newBaseRedisUnique(cfg *BaseConfig) (baseQueue, error) { + return newBaseRedisGeneric(cfg, true) +} + +func (q *baseRedis) PushItem(ctx context.Context, data []byte) error { + return backoffErr(ctx, backoffBegin, backoffUpper, time.After(pushBlockTime), func() (retry bool, err error) { + q.mu.Lock() + defer q.mu.Unlock() + + cnt, err := q.client.LLen(ctx, q.cfg.QueueFullName).Result() + if err != nil { + return false, err + } + if int(cnt) >= q.cfg.Length { + return true, nil + } + + if q.isUnique { + added, err := q.client.SAdd(ctx, q.cfg.SetFullName, data).Result() + if err != nil { + return false, err + } + if added == 0 { + return false, ErrAlreadyInQueue + } + } + return false, q.client.RPush(ctx, q.cfg.QueueFullName, data).Err() + }) +} + +func (q *baseRedis) PopItem(ctx context.Context) ([]byte, error) { + return backoffRetErr(ctx, backoffBegin, backoffUpper, infiniteTimerC, func() (retry bool, data []byte, err error) { + q.mu.Lock() + defer q.mu.Unlock() + + data, err = q.client.LPop(ctx, q.cfg.QueueFullName).Bytes() + if err == redis.Nil { + return true, nil, nil + } + if err != nil { + return true, nil, nil + } + if q.isUnique { + // the data has been popped, even if there is any error we can't do anything + _ = q.client.SRem(ctx, q.cfg.SetFullName, data).Err() + } + return false, data, err + }) +} + +func (q *baseRedis) HasItem(ctx context.Context, data []byte) (bool, error) { + q.mu.Lock() + defer q.mu.Unlock() + if !q.isUnique { + return false, nil + } + return q.client.SIsMember(ctx, q.cfg.SetFullName, data).Result() +} + +func (q *baseRedis) Len(ctx context.Context) (int, error) { + q.mu.Lock() + defer q.mu.Unlock() + cnt, err := q.client.LLen(ctx, q.cfg.QueueFullName).Result() + return int(cnt), err +} + +func (q *baseRedis) Close() error { + q.mu.Lock() + defer q.mu.Unlock() + return q.client.Close() +} + +func (q *baseRedis) RemoveAll(ctx context.Context) error { + q.mu.Lock() + defer q.mu.Unlock() + + c1 := q.client.Del(ctx, q.cfg.QueueFullName) + // the "set" must be cleared after the "list" because there is no transaction. + // it's better to have duplicate items than losing items. + c2 := q.client.Del(ctx, q.cfg.SetFullName) + if c1.Err() != nil { + return c1.Err() + } + if c2.Err() != nil { + return c2.Err() + } + return nil // actually, checking errors doesn't make sense here because the state could be out-of-sync +} diff --git a/modules/queue/base_redis_test.go b/modules/queue/base_redis_test.go new file mode 100644 index 00000000000..19fbccbc8fe --- /dev/null +++ b/modules/queue/base_redis_test.go @@ -0,0 +1,71 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "os" + "os/exec" + "testing" + "time" + + "code.gitea.io/gitea/modules/nosql" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func waitRedisReady(conn string, dur time.Duration) (ready bool) { + ctxTimed, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + for t := time.Now(); ; time.Sleep(50 * time.Millisecond) { + ret := nosql.GetManager().GetRedisClient(conn).Ping(ctxTimed) + if ret.Err() == nil { + return true + } + if time.Since(t) > dur { + return false + } + } +} + +func redisServerCmd(t *testing.T) *exec.Cmd { + redisServerProg, err := exec.LookPath("redis-server") + if err != nil { + return nil + } + c := &exec.Cmd{ + Path: redisServerProg, + Args: []string{redisServerProg, "--bind", "127.0.0.1", "--port", "6379"}, + Dir: t.TempDir(), + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + return c +} + +func TestBaseRedis(t *testing.T) { + var redisServer *exec.Cmd + defer func() { + if redisServer != nil { + _ = redisServer.Process.Signal(os.Interrupt) + _ = redisServer.Wait() + } + }() + if !waitRedisReady("redis://127.0.0.1:6379/0", 0) { + redisServer = redisServerCmd(t) + if redisServer == nil && os.Getenv("CI") == "" { + t.Skip("redis-server not found") + return + } + assert.NoError(t, redisServer.Start()) + if !assert.True(t, waitRedisReady("redis://127.0.0.1:6379/0", 5*time.Second), "start redis-server") { + return + } + } + + testQueueBasic(t, newBaseRedisSimple, toBaseConfig("baseRedis", setting.QueueSettings{Length: 10}), false) + testQueueBasic(t, newBaseRedisUnique, toBaseConfig("baseRedisUnique", setting.QueueSettings{Length: 10}), true) +} diff --git a/modules/queue/base_test.go b/modules/queue/base_test.go new file mode 100644 index 00000000000..c5bf526ae67 --- /dev/null +++ b/modules/queue/base_test.go @@ -0,0 +1,140 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error), cfg *BaseConfig, isUnique bool) { + t.Run(fmt.Sprintf("testQueueBasic-%s-unique:%v", cfg.ManagedName, isUnique), func(t *testing.T) { + q, err := newFn(cfg) + assert.NoError(t, err) + + ctx := context.Background() + _ = q.RemoveAll(ctx) + cnt, err := q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, 0, cnt) + + // push the first item + err = q.PushItem(ctx, []byte("foo")) + assert.NoError(t, err) + + cnt, err = q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, 1, cnt) + + // push a duplicate item + err = q.PushItem(ctx, []byte("foo")) + if !isUnique { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, ErrAlreadyInQueue) + } + + // check the duplicate item + cnt, err = q.Len(ctx) + assert.NoError(t, err) + has, err := q.HasItem(ctx, []byte("foo")) + assert.NoError(t, err) + if !isUnique { + assert.EqualValues(t, 2, cnt) + assert.EqualValues(t, false, has) // non-unique queues don't check for duplicates + } else { + assert.EqualValues(t, 1, cnt) + assert.EqualValues(t, true, has) + } + + // push another item + err = q.PushItem(ctx, []byte("bar")) + assert.NoError(t, err) + + // pop the first item (and the duplicate if non-unique) + it, err := q.PopItem(ctx) + assert.NoError(t, err) + assert.EqualValues(t, "foo", string(it)) + + if !isUnique { + it, err = q.PopItem(ctx) + assert.NoError(t, err) + assert.EqualValues(t, "foo", string(it)) + } + + // pop another item + it, err = q.PopItem(ctx) + assert.NoError(t, err) + assert.EqualValues(t, "bar", string(it)) + + // pop an empty queue (timeout, cancel) + ctxTimed, cancel := context.WithTimeout(ctx, 10*time.Millisecond) + it, err = q.PopItem(ctxTimed) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Nil(t, it) + cancel() + + ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) + cancel() + it, err = q.PopItem(ctxTimed) + assert.ErrorIs(t, err, context.Canceled) + assert.Nil(t, it) + + // test blocking push if queue is full + for i := 0; i < cfg.Length; i++ { + err = q.PushItem(ctx, []byte(fmt.Sprintf("item-%d", i))) + assert.NoError(t, err) + } + ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) + err = q.PushItem(ctxTimed, []byte("item-full")) + assert.ErrorIs(t, err, context.DeadlineExceeded) + cancel() + + // test blocking push if queue is full (with custom pushBlockTime) + oldPushBlockTime := pushBlockTime + timeStart := time.Now() + pushBlockTime = 30 * time.Millisecond + err = q.PushItem(ctx, []byte("item-full")) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.True(t, time.Since(timeStart) >= pushBlockTime*2/3) + pushBlockTime = oldPushBlockTime + + // remove all + cnt, err = q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, cfg.Length, cnt) + + _ = q.RemoveAll(ctx) + + cnt, err = q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, 0, cnt) + }) +} + +func TestBaseDummy(t *testing.T) { + q, err := newBaseDummy(&BaseConfig{}, true) + assert.NoError(t, err) + + ctx := context.Background() + assert.NoError(t, q.PushItem(ctx, []byte("foo"))) + + cnt, err := q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, 0, cnt) + + has, err := q.HasItem(ctx, []byte("foo")) + assert.NoError(t, err) + assert.False(t, has) + + it, err := q.PopItem(ctx) + assert.NoError(t, err) + assert.Nil(t, it) + + assert.NoError(t, q.RemoveAll(ctx)) +} diff --git a/modules/queue/bytefifo.go b/modules/queue/bytefifo.go deleted file mode 100644 index c33b79426ed..00000000000 --- a/modules/queue/bytefifo.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import "context" - -// ByteFIFO defines a FIFO that takes a byte array -type ByteFIFO interface { - // Len returns the length of the fifo - Len(ctx context.Context) int64 - // PushFunc pushes data to the end of the fifo and calls the callback if it is added - PushFunc(ctx context.Context, data []byte, fn func() error) error - // Pop pops data from the start of the fifo - Pop(ctx context.Context) ([]byte, error) - // Close this fifo - Close() error - // PushBack pushes data back to the top of the fifo - PushBack(ctx context.Context, data []byte) error -} - -// UniqueByteFIFO defines a FIFO that Uniques its contents -type UniqueByteFIFO interface { - ByteFIFO - // Has returns whether the fifo contains this data - Has(ctx context.Context, data []byte) (bool, error) -} - -var _ ByteFIFO = &DummyByteFIFO{} - -// DummyByteFIFO represents a dummy fifo -type DummyByteFIFO struct{} - -// PushFunc returns nil -func (*DummyByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - return nil -} - -// Pop returns nil -func (*DummyByteFIFO) Pop(ctx context.Context) ([]byte, error) { - return []byte{}, nil -} - -// Close returns nil -func (*DummyByteFIFO) Close() error { - return nil -} - -// Len is always 0 -func (*DummyByteFIFO) Len(ctx context.Context) int64 { - return 0 -} - -// PushBack pushes data back to the top of the fifo -func (*DummyByteFIFO) PushBack(ctx context.Context, data []byte) error { - return nil -} - -var _ UniqueByteFIFO = &DummyUniqueByteFIFO{} - -// DummyUniqueByteFIFO represents a dummy unique fifo -type DummyUniqueByteFIFO struct { - DummyByteFIFO -} - -// Has always returns false -func (*DummyUniqueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { - return false, nil -} diff --git a/modules/queue/config.go b/modules/queue/config.go new file mode 100644 index 00000000000..c5bc16b6f08 --- /dev/null +++ b/modules/queue/config.go @@ -0,0 +1,36 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "code.gitea.io/gitea/modules/setting" +) + +type BaseConfig struct { + ManagedName string + DataFullDir string // the caller must prepare an absolute path + + ConnStr string + Length int + + QueueFullName, SetFullName string +} + +func toBaseConfig(managedName string, queueSetting setting.QueueSettings) *BaseConfig { + baseConfig := &BaseConfig{ + ManagedName: managedName, + DataFullDir: queueSetting.Datadir, + + ConnStr: queueSetting.ConnStr, + Length: queueSetting.Length, + } + + // queue name and set name + baseConfig.QueueFullName = managedName + queueSetting.QueueName + baseConfig.SetFullName = baseConfig.QueueFullName + queueSetting.SetName + if baseConfig.SetFullName == baseConfig.QueueFullName { + baseConfig.SetFullName += "_unique" + } + return baseConfig +} diff --git a/modules/queue/helper.go b/modules/queue/helper.go deleted file mode 100644 index c6fb9447b70..00000000000 --- a/modules/queue/helper.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "reflect" - - "code.gitea.io/gitea/modules/json" -) - -// Mappable represents an interface that can MapTo another interface -type Mappable interface { - MapTo(v interface{}) error -} - -// toConfig will attempt to convert a given configuration cfg into the provided exemplar type. -// -// It will tolerate the cfg being passed as a []byte or string of a json representation of the -// exemplar or the correct type of the exemplar itself -func toConfig(exemplar, cfg interface{}) (interface{}, error) { - // First of all check if we've got the same type as the exemplar - if so it's all fine. - if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) { - return cfg, nil - } - - // Now if not - does it provide a MapTo function we can try? - if mappable, ok := cfg.(Mappable); ok { - newVal := reflect.New(reflect.TypeOf(exemplar)) - if err := mappable.MapTo(newVal.Interface()); err == nil { - return newVal.Elem().Interface(), nil - } - // MapTo has failed us ... let's try the json route ... - } - - // OK we've been passed a byte array right? - configBytes, ok := cfg.([]byte) - if !ok { - // oh ... it's a string then? - var configStr string - - configStr, ok = cfg.(string) - configBytes = []byte(configStr) - } - if !ok { - // hmm ... can we marshal it to json? - var err error - configBytes, err = json.Marshal(cfg) - ok = err == nil - } - if !ok { - // no ... we've tried hard enough at this point - throw an error! - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - // OK unmarshal the byte array into a new copy of the exemplar - newVal := reflect.New(reflect.TypeOf(exemplar)) - if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil { - // If we can't unmarshal it then return an error! - return nil, ErrInvalidConfiguration{cfg: cfg, err: err} - } - return newVal.Elem().Interface(), nil -} - -// unmarshalAs will attempt to unmarshal provided bytes as the provided exemplar -func unmarshalAs(bs []byte, exemplar interface{}) (data Data, err error) { - if exemplar != nil { - t := reflect.TypeOf(exemplar) - n := reflect.New(t) - ne := n.Elem() - err = json.Unmarshal(bs, ne.Addr().Interface()) - data = ne.Interface().(Data) - } else { - err = json.Unmarshal(bs, &data) - } - return data, err -} - -// assignableTo will check if provided data is assignable to the same type as the exemplar -// if the provided exemplar is nil then it will always return true -func assignableTo(data Data, exemplar interface{}) bool { - if exemplar == nil { - return true - } - - // Assert data is of same type as exemplar - t := reflect.TypeOf(data) - exemplarType := reflect.TypeOf(exemplar) - - return t.AssignableTo(exemplarType) && data != nil -} diff --git a/modules/queue/manager.go b/modules/queue/manager.go index 6975e029077..95b3bad57bf 100644 --- a/modules/queue/manager.go +++ b/modules/queue/manager.go @@ -5,457 +5,109 @@ package queue import ( "context" - "fmt" - "reflect" - "sort" - "strings" "sync" "time" - "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" ) +// Manager is a manager for the queues created by "CreateXxxQueue" functions, these queues are called "managed queues". +type Manager struct { + mu sync.Mutex + + qidCounter int64 + Queues map[int64]ManagedWorkerPoolQueue +} + +type ManagedWorkerPoolQueue interface { + GetName() string + GetType() string + GetItemTypeName() string + GetWorkerNumber() int + GetWorkerActiveNumber() int + GetWorkerMaxNumber() int + SetWorkerMaxNumber(num int) + GetQueueItemNumber() int + + // FlushWithContext tries to make the handler process all items in the queue synchronously. + // It is for testing purpose only. It's not designed to be used in a cluster. + FlushWithContext(ctx context.Context, timeout time.Duration) error + + // RemoveAllItems removes all items in the base queue (on-the-fly items are not affected) + RemoveAllItems(ctx context.Context) error +} + var manager *Manager -// Manager is a queue manager -type Manager struct { - mutex sync.Mutex - - counter int64 - Queues map[int64]*ManagedQueue -} - -// ManagedQueue represents a working queue with a Pool of workers. -// -// Although a ManagedQueue should really represent a Queue this does not -// necessarily have to be the case. This could be used to describe any queue.WorkerPool. -type ManagedQueue struct { - mutex sync.Mutex - QID int64 - Type Type - Name string - Configuration interface{} - ExemplarType string - Managed interface{} - counter int64 - PoolWorkers map[int64]*PoolWorkers -} - -// Flushable represents a pool or queue that is flushable -type Flushable interface { - // Flush will add a flush worker to the pool - the worker should be autoregistered with the manager - Flush(time.Duration) error - // FlushWithContext is very similar to Flush - // NB: The worker will not be registered with the manager. - FlushWithContext(ctx context.Context) error - // IsEmpty will return if the managed pool is empty and has no work - IsEmpty() bool -} - -// Pausable represents a pool or queue that is Pausable -type Pausable interface { - // IsPaused will return if the pool or queue is paused - IsPaused() bool - // Pause will pause the pool or queue - Pause() - // Resume will resume the pool or queue - Resume() - // IsPausedIsResumed will return a bool indicating if the pool or queue is paused and a channel that will be closed when it is resumed - IsPausedIsResumed() (paused, resumed <-chan struct{}) -} - -// ManagedPool is a simple interface to get certain details from a worker pool -type ManagedPool interface { - // AddWorkers adds a number of worker as group to the pool with the provided timeout. A CancelFunc is provided to cancel the group - AddWorkers(number int, timeout time.Duration) context.CancelFunc - // NumberOfWorkers returns the total number of workers in the pool - NumberOfWorkers() int - // MaxNumberOfWorkers returns the maximum number of workers the pool can dynamically grow to - MaxNumberOfWorkers() int - // SetMaxNumberOfWorkers sets the maximum number of workers the pool can dynamically grow to - SetMaxNumberOfWorkers(int) - // BoostTimeout returns the current timeout for worker groups created during a boost - BoostTimeout() time.Duration - // BlockTimeout returns the timeout the internal channel can block for before a boost would occur - BlockTimeout() time.Duration - // BoostWorkers sets the number of workers to be created during a boost - BoostWorkers() int - // SetPoolSettings sets the user updatable settings for the pool - SetPoolSettings(maxNumberOfWorkers, boostWorkers int, timeout time.Duration) - // NumberInQueue returns the total number of items in the pool - NumberInQueue() int64 - // Done returns a channel that will be closed when the Pool's baseCtx is closed - Done() <-chan struct{} -} - -// ManagedQueueList implements the sort.Interface -type ManagedQueueList []*ManagedQueue - -// PoolWorkers represents a group of workers working on a queue -type PoolWorkers struct { - PID int64 - Workers int - Start time.Time - Timeout time.Time - HasTimeout bool - Cancel context.CancelFunc - IsFlusher bool -} - -// PoolWorkersList implements the sort.Interface for PoolWorkers -type PoolWorkersList []*PoolWorkers - func init() { - _ = GetManager() + manager = &Manager{ + Queues: make(map[int64]ManagedWorkerPoolQueue), + } } -// GetManager returns a Manager and initializes one as singleton if there's none yet func GetManager() *Manager { - if manager == nil { - manager = &Manager{ - Queues: make(map[int64]*ManagedQueue), - } - } return manager } -// Add adds a queue to this manager -func (m *Manager) Add(managed interface{}, - t Type, - configuration, - exemplar interface{}, -) int64 { - cfg, _ := json.Marshal(configuration) - mq := &ManagedQueue{ - Type: t, - Configuration: string(cfg), - ExemplarType: reflect.TypeOf(exemplar).String(), - PoolWorkers: make(map[int64]*PoolWorkers), - Managed: managed, - } - m.mutex.Lock() - m.counter++ - mq.QID = m.counter - mq.Name = fmt.Sprintf("queue-%d", mq.QID) - if named, ok := managed.(Named); ok { - name := named.Name() - if len(name) > 0 { - mq.Name = name - } - } - m.Queues[mq.QID] = mq - m.mutex.Unlock() - log.Trace("Queue Manager registered: %s (QID: %d)", mq.Name, mq.QID) - return mq.QID +func (m *Manager) AddManagedQueue(managed ManagedWorkerPoolQueue) { + m.mu.Lock() + defer m.mu.Unlock() + m.qidCounter++ + m.Queues[m.qidCounter] = managed } -// Remove a queue from the Manager -func (m *Manager) Remove(qid int64) { - m.mutex.Lock() - delete(m.Queues, qid) - m.mutex.Unlock() - log.Trace("Queue Manager removed: QID: %d", qid) -} - -// GetManagedQueue by qid -func (m *Manager) GetManagedQueue(qid int64) *ManagedQueue { - m.mutex.Lock() - defer m.mutex.Unlock() +func (m *Manager) GetManagedQueue(qid int64) ManagedWorkerPoolQueue { + m.mu.Lock() + defer m.mu.Unlock() return m.Queues[qid] } -// FlushAll flushes all the flushable queues attached to this manager -func (m *Manager) FlushAll(baseCtx context.Context, timeout time.Duration) error { - var ctx context.Context - var cancel context.CancelFunc - start := time.Now() - end := start - hasTimeout := false - if timeout > 0 { - ctx, cancel = context.WithTimeout(baseCtx, timeout) - end = start.Add(timeout) - hasTimeout = true - } else { - ctx, cancel = context.WithCancel(baseCtx) - } - defer cancel() +func (m *Manager) ManagedQueues() map[int64]ManagedWorkerPoolQueue { + m.mu.Lock() + defer m.mu.Unlock() - for { - select { - case <-ctx.Done(): - mqs := m.ManagedQueues() - nonEmptyQueues := []string{} - for _, mq := range mqs { - if !mq.IsEmpty() { - nonEmptyQueues = append(nonEmptyQueues, mq.Name) - } - } - if len(nonEmptyQueues) > 0 { - return fmt.Errorf("flush timeout with non-empty queues: %s", strings.Join(nonEmptyQueues, ", ")) - } - return nil - default: + queues := make(map[int64]ManagedWorkerPoolQueue, len(m.Queues)) + for k, v := range m.Queues { + queues[k] = v + } + return queues +} + +// FlushAll tries to make all managed queues process all items synchronously, until timeout or the queue is empty. +// It is for testing purpose only. It's not designed to be used in a cluster. +func (m *Manager) FlushAll(ctx context.Context, timeout time.Duration) error { + var finalErr error + qs := m.ManagedQueues() + for _, q := range qs { + if err := q.FlushWithContext(ctx, timeout); err != nil { + finalErr = err // TODO: in Go 1.20: errors.Join } - mqs := m.ManagedQueues() - log.Debug("Found %d Managed Queues", len(mqs)) - wg := sync.WaitGroup{} - wg.Add(len(mqs)) - allEmpty := true - for _, mq := range mqs { - if mq.IsEmpty() { - wg.Done() - continue - } - if pausable, ok := mq.Managed.(Pausable); ok { - // no point flushing paused queues - if pausable.IsPaused() { - wg.Done() - continue - } - } - if pool, ok := mq.Managed.(ManagedPool); ok { - // No point into flushing pools when their base's ctx is already done. - select { - case <-pool.Done(): - wg.Done() - continue - default: - } - } - - allEmpty = false - if flushable, ok := mq.Managed.(Flushable); ok { - log.Debug("Flushing (flushable) queue: %s", mq.Name) - go func(q *ManagedQueue) { - localCtx, localCtxCancel := context.WithCancel(ctx) - pid := q.RegisterWorkers(1, start, hasTimeout, end, localCtxCancel, true) - err := flushable.FlushWithContext(localCtx) - if err != nil && err != ctx.Err() { - cancel() - } - q.CancelWorkers(pid) - localCtxCancel() - wg.Done() - }(mq) - } else { - log.Debug("Queue: %s is non-empty but is not flushable", mq.Name) - wg.Done() - } - } - if allEmpty { - log.Debug("All queues are empty") - break - } - // Ensure there are always at least 100ms between loops but not more if we've actually been doing some flushing - // but don't delay cancellation here. - select { - case <-ctx.Done(): - case <-time.After(100 * time.Millisecond): - } - wg.Wait() } - return nil + return finalErr } -// ManagedQueues returns the managed queues -func (m *Manager) ManagedQueues() []*ManagedQueue { - m.mutex.Lock() - mqs := make([]*ManagedQueue, 0, len(m.Queues)) - for _, mq := range m.Queues { - mqs = append(mqs, mq) +// CreateSimpleQueue creates a simple queue from global setting config provider by name +func CreateSimpleQueue[T any](name string, handler HandlerFuncT[T]) *WorkerPoolQueue[T] { + return createWorkerPoolQueue(name, setting.CfgProvider, handler, false) +} + +// CreateUniqueQueue creates a unique queue from global setting config provider by name +func CreateUniqueQueue[T any](name string, handler HandlerFuncT[T]) *WorkerPoolQueue[T] { + return createWorkerPoolQueue(name, setting.CfgProvider, handler, true) +} + +func createWorkerPoolQueue[T any](name string, cfgProvider setting.ConfigProvider, handler HandlerFuncT[T], unique bool) *WorkerPoolQueue[T] { + queueSetting, err := setting.GetQueueSettings(cfgProvider, name) + if err != nil { + log.Error("Failed to get queue settings for %q: %v", name, err) + return nil } - m.mutex.Unlock() - sort.Sort(ManagedQueueList(mqs)) - return mqs -} - -// Workers returns the poolworkers -func (q *ManagedQueue) Workers() []*PoolWorkers { - q.mutex.Lock() - workers := make([]*PoolWorkers, 0, len(q.PoolWorkers)) - for _, worker := range q.PoolWorkers { - workers = append(workers, worker) + w, err := NewWorkerPoolQueueBySetting(name, queueSetting, handler, unique) + if err != nil { + log.Error("Failed to create queue %q: %v", name, err) + return nil } - q.mutex.Unlock() - sort.Sort(PoolWorkersList(workers)) - return workers -} - -// RegisterWorkers registers workers to this queue -func (q *ManagedQueue) RegisterWorkers(number int, start time.Time, hasTimeout bool, timeout time.Time, cancel context.CancelFunc, isFlusher bool) int64 { - q.mutex.Lock() - defer q.mutex.Unlock() - q.counter++ - q.PoolWorkers[q.counter] = &PoolWorkers{ - PID: q.counter, - Workers: number, - Start: start, - Timeout: timeout, - HasTimeout: hasTimeout, - Cancel: cancel, - IsFlusher: isFlusher, - } - return q.counter -} - -// CancelWorkers cancels pooled workers with pid -func (q *ManagedQueue) CancelWorkers(pid int64) { - q.mutex.Lock() - pw, ok := q.PoolWorkers[pid] - q.mutex.Unlock() - if !ok { - return - } - pw.Cancel() -} - -// RemoveWorkers deletes pooled workers with pid -func (q *ManagedQueue) RemoveWorkers(pid int64) { - q.mutex.Lock() - pw, ok := q.PoolWorkers[pid] - delete(q.PoolWorkers, pid) - q.mutex.Unlock() - if ok && pw.Cancel != nil { - pw.Cancel() - } -} - -// AddWorkers adds workers to the queue if it has registered an add worker function -func (q *ManagedQueue) AddWorkers(number int, timeout time.Duration) context.CancelFunc { - if pool, ok := q.Managed.(ManagedPool); ok { - // the cancel will be added to the pool workers description above - return pool.AddWorkers(number, timeout) - } - return nil -} - -// Flushable returns true if the queue is flushable -func (q *ManagedQueue) Flushable() bool { - _, ok := q.Managed.(Flushable) - return ok -} - -// Flush flushes the queue with a timeout -func (q *ManagedQueue) Flush(timeout time.Duration) error { - if flushable, ok := q.Managed.(Flushable); ok { - // the cancel will be added to the pool workers description above - return flushable.Flush(timeout) - } - return nil -} - -// IsEmpty returns if the queue is empty -func (q *ManagedQueue) IsEmpty() bool { - if flushable, ok := q.Managed.(Flushable); ok { - return flushable.IsEmpty() - } - return true -} - -// Pausable returns whether the queue is Pausable -func (q *ManagedQueue) Pausable() bool { - _, ok := q.Managed.(Pausable) - return ok -} - -// Pause pauses the queue -func (q *ManagedQueue) Pause() { - if pausable, ok := q.Managed.(Pausable); ok { - pausable.Pause() - } -} - -// IsPaused reveals if the queue is paused -func (q *ManagedQueue) IsPaused() bool { - if pausable, ok := q.Managed.(Pausable); ok { - return pausable.IsPaused() - } - return false -} - -// Resume resumes the queue -func (q *ManagedQueue) Resume() { - if pausable, ok := q.Managed.(Pausable); ok { - pausable.Resume() - } -} - -// NumberOfWorkers returns the number of workers in the queue -func (q *ManagedQueue) NumberOfWorkers() int { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.NumberOfWorkers() - } - return -1 -} - -// MaxNumberOfWorkers returns the maximum number of workers for the pool -func (q *ManagedQueue) MaxNumberOfWorkers() int { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.MaxNumberOfWorkers() - } - return 0 -} - -// BoostWorkers returns the number of workers for a boost -func (q *ManagedQueue) BoostWorkers() int { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.BoostWorkers() - } - return -1 -} - -// BoostTimeout returns the timeout of the next boost -func (q *ManagedQueue) BoostTimeout() time.Duration { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.BoostTimeout() - } - return 0 -} - -// BlockTimeout returns the timeout til the next boost -func (q *ManagedQueue) BlockTimeout() time.Duration { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.BlockTimeout() - } - return 0 -} - -// SetPoolSettings sets the setable boost values -func (q *ManagedQueue) SetPoolSettings(maxNumberOfWorkers, boostWorkers int, timeout time.Duration) { - if pool, ok := q.Managed.(ManagedPool); ok { - pool.SetPoolSettings(maxNumberOfWorkers, boostWorkers, timeout) - } -} - -// NumberInQueue returns the number of items in the queue -func (q *ManagedQueue) NumberInQueue() int64 { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.NumberInQueue() - } - return -1 -} - -func (l ManagedQueueList) Len() int { - return len(l) -} - -func (l ManagedQueueList) Less(i, j int) bool { - return l[i].Name < l[j].Name -} - -func (l ManagedQueueList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] -} - -func (l PoolWorkersList) Len() int { - return len(l) -} - -func (l PoolWorkersList) Less(i, j int) bool { - return l[i].Start.Before(l[j].Start) -} - -func (l PoolWorkersList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] + GetManager().AddManagedQueue(w) + return w } diff --git a/modules/queue/manager_test.go b/modules/queue/manager_test.go new file mode 100644 index 00000000000..50265e27b67 --- /dev/null +++ b/modules/queue/manager_test.go @@ -0,0 +1,124 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "path/filepath" + "testing" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestManager(t *testing.T) { + oldAppDataPath := setting.AppDataPath + setting.AppDataPath = t.TempDir() + defer func() { + setting.AppDataPath = oldAppDataPath + }() + + newQueueFromConfig := func(name, cfg string) (*WorkerPoolQueue[int], error) { + cfgProvider, err := setting.NewConfigProviderFromData(cfg) + if err != nil { + return nil, err + } + qs, err := setting.GetQueueSettings(cfgProvider, name) + if err != nil { + return nil, err + } + return NewWorkerPoolQueueBySetting(name, qs, func(s ...int) (unhandled []int) { return nil }, false) + } + + // test invalid CONN_STR + _, err := newQueueFromConfig("default", ` +[queue] +DATADIR = temp-dir +CONN_STR = redis:// +`) + assert.ErrorContains(t, err, "invalid leveldb connection string") + + // test default config + q, err := newQueueFromConfig("default", "") + assert.NoError(t, err) + assert.Equal(t, "default", q.GetName()) + assert.Equal(t, "level", q.GetType()) + assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/common"), q.baseConfig.DataFullDir) + assert.Equal(t, 100, q.baseConfig.Length) + assert.Equal(t, 20, q.batchLength) + assert.Equal(t, "", q.baseConfig.ConnStr) + assert.Equal(t, "default_queue", q.baseConfig.QueueFullName) + assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName) + assert.Equal(t, 10, q.GetWorkerMaxNumber()) + assert.Equal(t, 0, q.GetWorkerNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 0, q.GetQueueItemNumber()) + assert.Equal(t, "int", q.GetItemTypeName()) + + // test inherited config + cfgProvider, err := setting.NewConfigProviderFromData(` +[queue] +TYPE = channel +DATADIR = queues/dir1 +LENGTH = 100 +BATCH_LENGTH = 20 +CONN_STR = "addrs=127.0.0.1:6379 db=0" +QUEUE_NAME = _queue1 + +[queue.sub] +TYPE = level +DATADIR = queues/dir2 +LENGTH = 102 +BATCH_LENGTH = 22 +CONN_STR = +QUEUE_NAME = _q2 +SET_NAME = _u2 +MAX_WORKERS = 2 +`) + + assert.NoError(t, err) + + q1 := createWorkerPoolQueue[string]("no-such", cfgProvider, nil, false) + assert.Equal(t, "no-such", q1.GetName()) + assert.Equal(t, "dummy", q1.GetType()) // no handler, so it becomes dummy + assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir1"), q1.baseConfig.DataFullDir) + assert.Equal(t, 100, q1.baseConfig.Length) + assert.Equal(t, 20, q1.batchLength) + assert.Equal(t, "addrs=127.0.0.1:6379 db=0", q1.baseConfig.ConnStr) + assert.Equal(t, "no-such_queue1", q1.baseConfig.QueueFullName) + assert.Equal(t, "no-such_queue1_unique", q1.baseConfig.SetFullName) + assert.Equal(t, 10, q1.GetWorkerMaxNumber()) + assert.Equal(t, 0, q1.GetWorkerNumber()) + assert.Equal(t, 0, q1.GetWorkerActiveNumber()) + assert.Equal(t, 0, q1.GetQueueItemNumber()) + assert.Equal(t, "string", q1.GetItemTypeName()) + qid1 := GetManager().qidCounter + + q2 := createWorkerPoolQueue("sub", cfgProvider, func(s ...int) (unhandled []int) { return nil }, false) + assert.Equal(t, "sub", q2.GetName()) + assert.Equal(t, "level", q2.GetType()) + assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir2"), q2.baseConfig.DataFullDir) + assert.Equal(t, 102, q2.baseConfig.Length) + assert.Equal(t, 22, q2.batchLength) + assert.Equal(t, "", q2.baseConfig.ConnStr) + assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName) + assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName) + assert.Equal(t, 2, q2.GetWorkerMaxNumber()) + assert.Equal(t, 0, q2.GetWorkerNumber()) + assert.Equal(t, 0, q2.GetWorkerActiveNumber()) + assert.Equal(t, 0, q2.GetQueueItemNumber()) + assert.Equal(t, "int", q2.GetItemTypeName()) + qid2 := GetManager().qidCounter + + assert.Equal(t, q1, GetManager().ManagedQueues()[qid1]) + + GetManager().GetManagedQueue(qid1).SetWorkerMaxNumber(120) + assert.Equal(t, 120, q1.workerMaxNum) + + stop := runWorkerPoolQueue(q2) + assert.NoError(t, GetManager().GetManagedQueue(qid2).FlushWithContext(context.Background(), 0)) + assert.NoError(t, GetManager().FlushAll(context.Background(), 0)) + stop() +} diff --git a/modules/queue/queue.go b/modules/queue/queue.go index 22ee64f8e20..0ab8dd4ae47 100644 --- a/modules/queue/queue.go +++ b/modules/queue/queue.go @@ -1,201 +1,31 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT +// Package queue implements a specialized queue system for Gitea. +// +// There are two major kinds of concepts: +// +// * The "base queue": channel, level, redis: +// - They have the same abstraction, the same interface, and they are tested by the same testing code. +// - The dummy(immediate) queue is special, it's not a real queue, it's only used as a no-op queue or a testing queue. +// +// * The WorkerPoolQueue: it uses the "base queue" to provide "worker pool" function. +// - It calls the "handler" to process the data in the base queue. +// - Its "Push" function doesn't block forever, +// it will return an error if the queue is full after the timeout. +// +// A queue can be "simple" or "unique". A unique queue will try to avoid duplicate items. +// Unique queue's "Has" function can be used to check whether an item is already in the queue, +// although it's not 100% reliable due to there is no proper transaction support. +// Simple queue's "Has" function always returns "has=false". +// +// The HandlerFuncT function is called by the WorkerPoolQueue to process the data in the base queue. +// If the handler returns "unhandled" items, they will be re-queued to the base queue after a slight delay, +// in case the item processor (eg: document indexer) is not available. package queue -import ( - "context" - "fmt" - "time" -) +import "code.gitea.io/gitea/modules/util" -// ErrInvalidConfiguration is called when there is invalid configuration for a queue -type ErrInvalidConfiguration struct { - cfg interface{} - err error -} +type HandlerFuncT[T any] func(...T) (unhandled []T) -func (err ErrInvalidConfiguration) Error() string { - if err.err != nil { - return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err) - } - return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg) -} - -// IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration -func IsErrInvalidConfiguration(err error) bool { - _, ok := err.(ErrInvalidConfiguration) - return ok -} - -// Type is a type of Queue -type Type string - -// Data defines an type of queuable data -type Data interface{} - -// HandlerFunc is a function that takes a variable amount of data and processes it -type HandlerFunc func(...Data) (unhandled []Data) - -// NewQueueFunc is a function that creates a queue -type NewQueueFunc func(handler HandlerFunc, config, exemplar interface{}) (Queue, error) - -// Shutdownable represents a queue that can be shutdown -type Shutdownable interface { - Shutdown() - Terminate() -} - -// Named represents a queue with a name -type Named interface { - Name() string -} - -// Queue defines an interface of a queue-like item -// -// Queues will handle their own contents in the Run method -type Queue interface { - Flushable - Run(atShutdown, atTerminate func(func())) - Push(Data) error -} - -// PushBackable queues can be pushed back to -type PushBackable interface { - // PushBack pushes data back to the top of the fifo - PushBack(Data) error -} - -// DummyQueueType is the type for the dummy queue -const DummyQueueType Type = "dummy" - -// NewDummyQueue creates a new DummyQueue -func NewDummyQueue(handler HandlerFunc, opts, exemplar interface{}) (Queue, error) { - return &DummyQueue{}, nil -} - -// DummyQueue represents an empty queue -type DummyQueue struct{} - -// Run does nothing -func (*DummyQueue) Run(_, _ func(func())) {} - -// Push fakes a push of data to the queue -func (*DummyQueue) Push(Data) error { - return nil -} - -// PushFunc fakes a push of data to the queue with a function. The function is never run. -func (*DummyQueue) PushFunc(Data, func() error) error { - return nil -} - -// Has always returns false as this queue never does anything -func (*DummyQueue) Has(Data) (bool, error) { - return false, nil -} - -// Flush always returns nil -func (*DummyQueue) Flush(time.Duration) error { - return nil -} - -// FlushWithContext always returns nil -func (*DummyQueue) FlushWithContext(context.Context) error { - return nil -} - -// IsEmpty asserts that the queue is empty -func (*DummyQueue) IsEmpty() bool { - return true -} - -// ImmediateType is the type to execute the function when push -const ImmediateType Type = "immediate" - -// NewImmediate creates a new false queue to execute the function when push -func NewImmediate(handler HandlerFunc, opts, exemplar interface{}) (Queue, error) { - return &Immediate{ - handler: handler, - }, nil -} - -// Immediate represents an direct execution queue -type Immediate struct { - handler HandlerFunc -} - -// Run does nothing -func (*Immediate) Run(_, _ func(func())) {} - -// Push fakes a push of data to the queue -func (q *Immediate) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushFunc fakes a push of data to the queue with a function. The function is never run. -func (q *Immediate) PushFunc(data Data, f func() error) error { - if f != nil { - if err := f(); err != nil { - return err - } - } - q.handler(data) - return nil -} - -// Has always returns false as this queue never does anything -func (*Immediate) Has(Data) (bool, error) { - return false, nil -} - -// Flush always returns nil -func (*Immediate) Flush(time.Duration) error { - return nil -} - -// FlushWithContext always returns nil -func (*Immediate) FlushWithContext(context.Context) error { - return nil -} - -// IsEmpty asserts that the queue is empty -func (*Immediate) IsEmpty() bool { - return true -} - -var queuesMap = map[Type]NewQueueFunc{ - DummyQueueType: NewDummyQueue, - ImmediateType: NewImmediate, -} - -// RegisteredTypes provides the list of requested types of queues -func RegisteredTypes() []Type { - types := make([]Type, len(queuesMap)) - i := 0 - for key := range queuesMap { - types[i] = key - i++ - } - return types -} - -// RegisteredTypesAsString provides the list of requested types of queues -func RegisteredTypesAsString() []string { - types := make([]string, len(queuesMap)) - i := 0 - for key := range queuesMap { - types[i] = string(key) - i++ - } - return types -} - -// NewQueue takes a queue Type, HandlerFunc, some options and possibly an exemplar and returns a Queue or an error -func NewQueue(queueType Type, handlerFunc HandlerFunc, opts, exemplar interface{}) (Queue, error) { - newFn, ok := queuesMap[queueType] - if !ok { - return nil, fmt.Errorf("unsupported queue type: %v", queueType) - } - return newFn(handlerFunc, opts, exemplar) -} +var ErrAlreadyInQueue = util.NewAlreadyExistErrorf("already in queue") diff --git a/modules/queue/queue_bytefifo.go b/modules/queue/queue_bytefifo.go deleted file mode 100644 index ee00a5428ab..00000000000 --- a/modules/queue/queue_bytefifo.go +++ /dev/null @@ -1,419 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" -) - -// ByteFIFOQueueConfiguration is the configuration for a ByteFIFOQueue -type ByteFIFOQueueConfiguration struct { - WorkerPoolConfiguration - Workers int - WaitOnEmpty bool -} - -var _ Queue = &ByteFIFOQueue{} - -// ByteFIFOQueue is a Queue formed from a ByteFIFO and WorkerPool -type ByteFIFOQueue struct { - *WorkerPool - byteFIFO ByteFIFO - typ Type - shutdownCtx context.Context - shutdownCtxCancel context.CancelFunc - terminateCtx context.Context - terminateCtxCancel context.CancelFunc - exemplar interface{} - workers int - name string - lock sync.Mutex - waitOnEmpty bool - pushed chan struct{} -} - -// NewByteFIFOQueue creates a new ByteFIFOQueue -func NewByteFIFOQueue(typ Type, byteFIFO ByteFIFO, handle HandlerFunc, cfg, exemplar interface{}) (*ByteFIFOQueue, error) { - configInterface, err := toConfig(ByteFIFOQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(ByteFIFOQueueConfiguration) - - terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) - shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) - - q := &ByteFIFOQueue{ - byteFIFO: byteFIFO, - typ: typ, - shutdownCtx: shutdownCtx, - shutdownCtxCancel: shutdownCtxCancel, - terminateCtx: terminateCtx, - terminateCtxCancel: terminateCtxCancel, - exemplar: exemplar, - workers: config.Workers, - name: config.Name, - waitOnEmpty: config.WaitOnEmpty, - pushed: make(chan struct{}, 1), - } - q.WorkerPool = NewWorkerPool(func(data ...Data) (failed []Data) { - for _, unhandled := range handle(data...) { - if fail := q.PushBack(unhandled); fail != nil { - failed = append(failed, fail) - } - } - return failed - }, config.WorkerPoolConfiguration) - - return q, nil -} - -// Name returns the name of this queue -func (q *ByteFIFOQueue) Name() string { - return q.name -} - -// Push pushes data to the fifo -func (q *ByteFIFOQueue) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushBack pushes data to the fifo -func (q *ByteFIFOQueue) PushBack(data Data) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - bs, err := json.Marshal(data) - if err != nil { - return err - } - defer func() { - select { - case q.pushed <- struct{}{}: - default: - } - }() - return q.byteFIFO.PushBack(q.terminateCtx, bs) -} - -// PushFunc pushes data to the fifo -func (q *ByteFIFOQueue) PushFunc(data Data, fn func() error) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - bs, err := json.Marshal(data) - if err != nil { - return err - } - defer func() { - select { - case q.pushed <- struct{}{}: - default: - } - }() - return q.byteFIFO.PushFunc(q.terminateCtx, bs, fn) -} - -// IsEmpty checks if the queue is empty -func (q *ByteFIFOQueue) IsEmpty() bool { - q.lock.Lock() - defer q.lock.Unlock() - if !q.WorkerPool.IsEmpty() { - return false - } - return q.byteFIFO.Len(q.terminateCtx) == 0 -} - -// NumberInQueue returns the number in the queue -func (q *ByteFIFOQueue) NumberInQueue() int64 { - q.lock.Lock() - defer q.lock.Unlock() - return q.byteFIFO.Len(q.terminateCtx) + q.WorkerPool.NumberInQueue() -} - -// Flush flushes the ByteFIFOQueue -func (q *ByteFIFOQueue) Flush(timeout time.Duration) error { - select { - case q.pushed <- struct{}{}: - default: - } - return q.WorkerPool.Flush(timeout) -} - -// Run runs the bytefifo queue -func (q *ByteFIFOQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.baseCtx) - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - log.Debug("%s: %s Starting", q.typ, q.name) - - _ = q.AddWorkers(q.workers, 0) - - log.Trace("%s: %s Now running", q.typ, q.name) - q.readToChan() - - <-q.shutdownCtx.Done() - log.Trace("%s: %s Waiting til done", q.typ, q.name) - q.Wait() - - log.Trace("%s: %s Waiting til cleaned", q.typ, q.name) - q.CleanUp(q.terminateCtx) - q.terminateCtxCancel() -} - -const maxBackOffTime = time.Second * 3 - -func (q *ByteFIFOQueue) readToChan() { - // handle quick cancels - select { - case <-q.shutdownCtx.Done(): - // tell the pool to shutdown. - q.baseCtxCancel() - return - default: - } - - // Default backoff values - backOffTime := time.Millisecond * 100 - backOffTimer := time.NewTimer(0) - util.StopTimer(backOffTimer) - - paused, _ := q.IsPausedIsResumed() - -loop: - for { - select { - case <-paused: - log.Trace("Queue %s pausing", q.name) - _, resumed := q.IsPausedIsResumed() - - select { - case <-resumed: - paused, _ = q.IsPausedIsResumed() - log.Trace("Queue %s resuming", q.name) - if q.HasNoWorkerScaling() { - log.Warn( - "Queue: %s is configured to be non-scaling and has no workers - this configuration is likely incorrect.\n"+ - "The queue will be paused to prevent data-loss with the assumption that you will add workers and unpause as required.", q.name) - q.Pause() - continue loop - } - case <-q.shutdownCtx.Done(): - // tell the pool to shutdown. - q.baseCtxCancel() - return - case data, ok := <-q.dataChan: - if !ok { - return - } - if err := q.PushBack(data); err != nil { - log.Error("Unable to push back data into queue %s", q.name) - } - atomic.AddInt64(&q.numInQueue, -1) - } - default: - } - - // empty the pushed channel - select { - case <-q.pushed: - default: - } - - err := q.doPop() - - util.StopTimer(backOffTimer) - - if err != nil { - if err == errQueueEmpty && q.waitOnEmpty { - log.Trace("%s: %s Waiting on Empty", q.typ, q.name) - - // reset the backoff time but don't set the timer - backOffTime = 100 * time.Millisecond - } else if err == errUnmarshal { - // reset the timer and backoff - backOffTime = 100 * time.Millisecond - backOffTimer.Reset(backOffTime) - } else { - // backoff - backOffTimer.Reset(backOffTime) - } - - // Need to Backoff - select { - case <-q.shutdownCtx.Done(): - // Oops we've been shutdown whilst backing off - // Make sure the worker pool is shutdown too - q.baseCtxCancel() - return - case <-q.pushed: - // Data has been pushed to the fifo (or flush has been called) - // reset the backoff time - backOffTime = 100 * time.Millisecond - continue loop - case <-backOffTimer.C: - // Calculate the next backoff time - backOffTime += backOffTime / 2 - if backOffTime > maxBackOffTime { - backOffTime = maxBackOffTime - } - continue loop - } - } - - // Reset the backoff time - backOffTime = 100 * time.Millisecond - - select { - case <-q.shutdownCtx.Done(): - // Oops we've been shutdown - // Make sure the worker pool is shutdown too - q.baseCtxCancel() - return - default: - continue loop - } - } -} - -var ( - errQueueEmpty = fmt.Errorf("empty queue") - errEmptyBytes = fmt.Errorf("empty bytes") - errUnmarshal = fmt.Errorf("failed to unmarshal") -) - -func (q *ByteFIFOQueue) doPop() error { - q.lock.Lock() - defer q.lock.Unlock() - bs, err := q.byteFIFO.Pop(q.shutdownCtx) - if err != nil { - if err == context.Canceled { - q.baseCtxCancel() - return err - } - log.Error("%s: %s Error on Pop: %v", q.typ, q.name, err) - return err - } - if len(bs) == 0 { - if q.waitOnEmpty && q.byteFIFO.Len(q.shutdownCtx) == 0 { - return errQueueEmpty - } - return errEmptyBytes - } - - data, err := unmarshalAs(bs, q.exemplar) - if err != nil { - log.Error("%s: %s Failed to unmarshal with error: %v", q.typ, q.name, err) - return errUnmarshal - } - - log.Trace("%s %s: Task found: %#v", q.typ, q.name, data) - q.WorkerPool.Push(data) - return nil -} - -// Shutdown processing from this queue -func (q *ByteFIFOQueue) Shutdown() { - log.Trace("%s: %s Shutting down", q.typ, q.name) - select { - case <-q.shutdownCtx.Done(): - return - default: - } - q.shutdownCtxCancel() - log.Debug("%s: %s Shutdown", q.typ, q.name) -} - -// IsShutdown returns a channel which is closed when this Queue is shutdown -func (q *ByteFIFOQueue) IsShutdown() <-chan struct{} { - return q.shutdownCtx.Done() -} - -// Terminate this queue and close the queue -func (q *ByteFIFOQueue) Terminate() { - log.Trace("%s: %s Terminating", q.typ, q.name) - q.Shutdown() - select { - case <-q.terminateCtx.Done(): - return - default: - } - if log.IsDebug() { - log.Debug("%s: %s Closing with %d tasks left in queue", q.typ, q.name, q.byteFIFO.Len(q.terminateCtx)) - } - q.terminateCtxCancel() - if err := q.byteFIFO.Close(); err != nil { - log.Error("Error whilst closing internal byte fifo in %s: %s: %v", q.typ, q.name, err) - } - q.baseCtxFinished() - log.Debug("%s: %s Terminated", q.typ, q.name) -} - -// IsTerminated returns a channel which is closed when this Queue is terminated -func (q *ByteFIFOQueue) IsTerminated() <-chan struct{} { - return q.terminateCtx.Done() -} - -var _ UniqueQueue = &ByteFIFOUniqueQueue{} - -// ByteFIFOUniqueQueue represents a UniqueQueue formed from a UniqueByteFifo -type ByteFIFOUniqueQueue struct { - ByteFIFOQueue -} - -// NewByteFIFOUniqueQueue creates a new ByteFIFOUniqueQueue -func NewByteFIFOUniqueQueue(typ Type, byteFIFO UniqueByteFIFO, handle HandlerFunc, cfg, exemplar interface{}) (*ByteFIFOUniqueQueue, error) { - configInterface, err := toConfig(ByteFIFOQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(ByteFIFOQueueConfiguration) - terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) - shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) - - q := &ByteFIFOUniqueQueue{ - ByteFIFOQueue: ByteFIFOQueue{ - byteFIFO: byteFIFO, - typ: typ, - shutdownCtx: shutdownCtx, - shutdownCtxCancel: shutdownCtxCancel, - terminateCtx: terminateCtx, - terminateCtxCancel: terminateCtxCancel, - exemplar: exemplar, - workers: config.Workers, - name: config.Name, - }, - } - q.WorkerPool = NewWorkerPool(func(data ...Data) (failed []Data) { - for _, unhandled := range handle(data...) { - if fail := q.PushBack(unhandled); fail != nil { - failed = append(failed, fail) - } - } - return failed - }, config.WorkerPoolConfiguration) - - return q, nil -} - -// Has checks if the provided data is in the queue -func (q *ByteFIFOUniqueQueue) Has(data Data) (bool, error) { - if !assignableTo(data, q.exemplar) { - return false, fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - bs, err := json.Marshal(data) - if err != nil { - return false, err - } - return q.byteFIFO.(UniqueByteFIFO).Has(q.terminateCtx, bs) -} diff --git a/modules/queue/queue_channel.go b/modules/queue/queue_channel.go deleted file mode 100644 index baac0973939..00000000000 --- a/modules/queue/queue_channel.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/log" -) - -// ChannelQueueType is the type for channel queue -const ChannelQueueType Type = "channel" - -// ChannelQueueConfiguration is the configuration for a ChannelQueue -type ChannelQueueConfiguration struct { - WorkerPoolConfiguration - Workers int -} - -// ChannelQueue implements Queue -// -// A channel queue is not persistable and does not shutdown or terminate cleanly -// It is basically a very thin wrapper around a WorkerPool -type ChannelQueue struct { - *WorkerPool - shutdownCtx context.Context - shutdownCtxCancel context.CancelFunc - terminateCtx context.Context - terminateCtxCancel context.CancelFunc - exemplar interface{} - workers int - name string -} - -// NewChannelQueue creates a memory channel queue -func NewChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(ChannelQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(ChannelQueueConfiguration) - if config.BatchLength == 0 { - config.BatchLength = 1 - } - - terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) - shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) - - queue := &ChannelQueue{ - shutdownCtx: shutdownCtx, - shutdownCtxCancel: shutdownCtxCancel, - terminateCtx: terminateCtx, - terminateCtxCancel: terminateCtxCancel, - exemplar: exemplar, - workers: config.Workers, - name: config.Name, - } - queue.WorkerPool = NewWorkerPool(func(data ...Data) []Data { - unhandled := handle(data...) - if len(unhandled) > 0 { - // We can only pushback to the channel if we're paused. - if queue.IsPaused() { - atomic.AddInt64(&queue.numInQueue, int64(len(unhandled))) - go func() { - for _, datum := range data { - queue.dataChan <- datum - } - }() - return nil - } - } - return unhandled - }, config.WorkerPoolConfiguration) - - queue.qid = GetManager().Add(queue, ChannelQueueType, config, exemplar) - return queue, nil -} - -// Run starts to run the queue -func (q *ChannelQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.baseCtx) - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - log.Debug("ChannelQueue: %s Starting", q.name) - _ = q.AddWorkers(q.workers, 0) -} - -// Push will push data into the queue -func (q *ChannelQueue) Push(data Data) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in queue: %s", data, q.exemplar, q.name) - } - q.WorkerPool.Push(data) - return nil -} - -// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager -func (q *ChannelQueue) Flush(timeout time.Duration) error { - if q.IsPaused() { - return nil - } - ctx, cancel := q.commonRegisterWorkers(1, timeout, true) - defer cancel() - return q.FlushWithContext(ctx) -} - -// Shutdown processing from this queue -func (q *ChannelQueue) Shutdown() { - q.lock.Lock() - defer q.lock.Unlock() - select { - case <-q.shutdownCtx.Done(): - log.Trace("ChannelQueue: %s Already Shutting down", q.name) - return - default: - } - log.Trace("ChannelQueue: %s Shutting down", q.name) - go func() { - log.Trace("ChannelQueue: %s Flushing", q.name) - // We can't use Cleanup here because that will close the channel - if err := q.FlushWithContext(q.terminateCtx); err != nil { - count := atomic.LoadInt64(&q.numInQueue) - if count > 0 { - log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name) - } - return - } - log.Debug("ChannelQueue: %s Flushed", q.name) - }() - q.shutdownCtxCancel() - log.Debug("ChannelQueue: %s Shutdown", q.name) -} - -// Terminate this queue and close the queue -func (q *ChannelQueue) Terminate() { - log.Trace("ChannelQueue: %s Terminating", q.name) - q.Shutdown() - select { - case <-q.terminateCtx.Done(): - return - default: - } - q.terminateCtxCancel() - q.baseCtxFinished() - log.Debug("ChannelQueue: %s Terminated", q.name) -} - -// Name returns the name of this queue -func (q *ChannelQueue) Name() string { - return q.name -} - -func init() { - queuesMap[ChannelQueueType] = NewChannelQueue -} diff --git a/modules/queue/queue_channel_test.go b/modules/queue/queue_channel_test.go deleted file mode 100644 index f9dae742e29..00000000000 --- a/modules/queue/queue_channel_test.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "os" - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/log" - - "github.com/stretchr/testify/assert" -) - -func TestChannelQueue(t *testing.T) { - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - nilFn := func(_ func()) {} - - queue, err := NewChannelQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 0, - MaxWorkers: 10, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 5, - Name: "TestChannelQueue", - }, - Workers: 0, - }, &testData{}) - assert.NoError(t, err) - - assert.Equal(t, 5, queue.(*ChannelQueue).WorkerPool.boostWorkers) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - go queue.Push(&test1) - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) -} - -func TestChannelQueue_Batch(t *testing.T) { - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - assert.True(t, len(data) == 2) - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - nilFn := func(_ func()) {} - - queue, err := NewChannelQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 2, - BlockTimeout: 0, - BoostTimeout: 0, - BoostWorkers: 0, - MaxWorkers: 10, - }, - Workers: 1, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - queue.Push(&test1) - go queue.Push(&test2) - - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - result2 := <-handleChan - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) -} - -func TestChannelQueue_Pause(t *testing.T) { - if os.Getenv("CI") != "" { - t.Skip("Skipping because test is flaky on CI") - } - lock := sync.Mutex{} - var queue Queue - var err error - pushBack := false - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - lock.Lock() - if pushBack { - if pausable, ok := queue.(Pausable); ok { - pausable.Pause() - } - lock.Unlock() - return data - } - lock.Unlock() - - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - queueShutdown := []func(){} - queueTerminate := []func(){} - - terminated := make(chan struct{}) - - queue, err = NewChannelQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 1, - BlockTimeout: 0, - BoostTimeout: 0, - BoostWorkers: 0, - MaxWorkers: 10, - }, - Workers: 1, - }, &testData{}) - assert.NoError(t, err) - - go func() { - queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) - close(terminated) - }() - - // Shutdown and Terminate in defer - defer func() { - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - lock.Lock() - log.Info("Finally terminating") - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - }() - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - queue.Push(&test1) - - pausable, ok := queue.(Pausable) - if !assert.True(t, ok) { - return - } - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - pausable.Pause() - - paused, _ := pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not paused") - return - } - - queue.Push(&test2) - - var result2 *testData - select { - case result2 = <-handleChan: - assert.Fail(t, "handler chan should be empty") - case <-time.After(100 * time.Millisecond): - } - - assert.Nil(t, result2) - - pausable.Resume() - _, resumed := pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - } - - select { - case result2 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test2") - } - - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - lock.Lock() - pushBack = true - lock.Unlock() - - _, resumed = pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not resumed") - return - } - - queue.Push(&test1) - paused, _ = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-handleChan: - assert.Fail(t, "handler chan should not contain test1") - return - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "queue should be paused") - return - } - - lock.Lock() - pushBack = false - lock.Unlock() - - paused, _ = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not paused") - return - } - - pausable.Resume() - _, resumed = pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - } - - select { - case result1 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test1") - } - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - queueShutdown = queueShutdown[:0] - lock.Unlock() - // Now shutdown the queue - for _, callback := range callbacks { - callback() - } - - // terminate the queue - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - queueShutdown = queueTerminate[:0] - lock.Unlock() - for _, callback := range callbacks { - callback() - } - select { - case <-terminated: - case <-time.After(10 * time.Second): - assert.Fail(t, "Queue should have terminated") - return - } -} diff --git a/modules/queue/queue_disk.go b/modules/queue/queue_disk.go deleted file mode 100644 index fbedb8e5b93..00000000000 --- a/modules/queue/queue_disk.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - - "code.gitea.io/gitea/modules/nosql" - - "gitea.com/lunny/levelqueue" -) - -// LevelQueueType is the type for level queue -const LevelQueueType Type = "level" - -// LevelQueueConfiguration is the configuration for a LevelQueue -type LevelQueueConfiguration struct { - ByteFIFOQueueConfiguration - DataDir string - ConnectionString string - QueueName string -} - -// LevelQueue implements a disk library queue -type LevelQueue struct { - *ByteFIFOQueue -} - -// NewLevelQueue creates a ledis local queue -func NewLevelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(LevelQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(LevelQueueConfiguration) - - if len(config.ConnectionString) == 0 { - config.ConnectionString = config.DataDir - } - config.WaitOnEmpty = true - - byteFIFO, err := NewLevelQueueByteFIFO(config.ConnectionString, config.QueueName) - if err != nil { - return nil, err - } - - byteFIFOQueue, err := NewByteFIFOQueue(LevelQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) - if err != nil { - return nil, err - } - - queue := &LevelQueue{ - ByteFIFOQueue: byteFIFOQueue, - } - queue.qid = GetManager().Add(queue, LevelQueueType, config, exemplar) - return queue, nil -} - -var _ ByteFIFO = &LevelQueueByteFIFO{} - -// LevelQueueByteFIFO represents a ByteFIFO formed from a LevelQueue -type LevelQueueByteFIFO struct { - internal *levelqueue.Queue - connection string -} - -// NewLevelQueueByteFIFO creates a ByteFIFO formed from a LevelQueue -func NewLevelQueueByteFIFO(connection, prefix string) (*LevelQueueByteFIFO, error) { - db, err := nosql.GetManager().GetLevelDB(connection) - if err != nil { - return nil, err - } - - internal, err := levelqueue.NewQueue(db, []byte(prefix), false) - if err != nil { - return nil, err - } - - return &LevelQueueByteFIFO{ - connection: connection, - internal: internal, - }, nil -} - -// PushFunc will push data into the fifo -func (fifo *LevelQueueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - if fn != nil { - if err := fn(); err != nil { - return err - } - } - return fifo.internal.LPush(data) -} - -// PushBack pushes data to the top of the fifo -func (fifo *LevelQueueByteFIFO) PushBack(ctx context.Context, data []byte) error { - return fifo.internal.RPush(data) -} - -// Pop pops data from the start of the fifo -func (fifo *LevelQueueByteFIFO) Pop(ctx context.Context) ([]byte, error) { - data, err := fifo.internal.RPop() - if err != nil && err != levelqueue.ErrNotFound { - return nil, err - } - return data, nil -} - -// Close this fifo -func (fifo *LevelQueueByteFIFO) Close() error { - err := fifo.internal.Close() - _ = nosql.GetManager().CloseLevelDB(fifo.connection) - return err -} - -// Len returns the length of the fifo -func (fifo *LevelQueueByteFIFO) Len(ctx context.Context) int64 { - return fifo.internal.Len() -} - -func init() { - queuesMap[LevelQueueType] = NewLevelQueue -} diff --git a/modules/queue/queue_disk_channel.go b/modules/queue/queue_disk_channel.go deleted file mode 100644 index 91f91f0dfc8..00000000000 --- a/modules/queue/queue_disk_channel.go +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/log" -) - -// PersistableChannelQueueType is the type for persistable queue -const PersistableChannelQueueType Type = "persistable-channel" - -// PersistableChannelQueueConfiguration is the configuration for a PersistableChannelQueue -type PersistableChannelQueueConfiguration struct { - Name string - DataDir string - BatchLength int - QueueLength int - Timeout time.Duration - MaxAttempts int - Workers int - MaxWorkers int - BlockTimeout time.Duration - BoostTimeout time.Duration - BoostWorkers int -} - -// PersistableChannelQueue wraps a channel queue and level queue together -// The disk level queue will be used to store data at shutdown and terminate - and will be restored -// on start up. -type PersistableChannelQueue struct { - channelQueue *ChannelQueue - delayedStarter - lock sync.Mutex - closed chan struct{} -} - -// NewPersistableChannelQueue creates a wrapped batched channel queue with persistable level queue backend when shutting down -// This differs from a wrapped queue in that the persistent queue is only used to persist at shutdown/terminate -func NewPersistableChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(PersistableChannelQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(PersistableChannelQueueConfiguration) - - queue := &PersistableChannelQueue{ - closed: make(chan struct{}), - } - - wrappedHandle := func(data ...Data) (failed []Data) { - for _, unhandled := range handle(data...) { - if fail := queue.PushBack(unhandled); fail != nil { - failed = append(failed, fail) - } - } - return failed - } - - channelQueue, err := NewChannelQueue(wrappedHandle, ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: config.QueueLength, - BatchLength: config.BatchLength, - BlockTimeout: config.BlockTimeout, - BoostTimeout: config.BoostTimeout, - BoostWorkers: config.BoostWorkers, - MaxWorkers: config.MaxWorkers, - Name: config.Name + "-channel", - }, - Workers: config.Workers, - }, exemplar) - if err != nil { - return nil, err - } - - // the level backend only needs temporary workers to catch up with the previously dropped work - levelCfg := LevelQueueConfiguration{ - ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: config.QueueLength, - BatchLength: config.BatchLength, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 1, - MaxWorkers: 5, - Name: config.Name + "-level", - }, - Workers: 0, - }, - DataDir: config.DataDir, - QueueName: config.Name + "-level", - } - - levelQueue, err := NewLevelQueue(wrappedHandle, levelCfg, exemplar) - if err == nil { - queue.channelQueue = channelQueue.(*ChannelQueue) - queue.delayedStarter = delayedStarter{ - internal: levelQueue.(*LevelQueue), - name: config.Name, - } - _ = GetManager().Add(queue, PersistableChannelQueueType, config, exemplar) - return queue, nil - } - if IsErrInvalidConfiguration(err) { - // Retrying ain't gonna make this any better... - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - queue.channelQueue = channelQueue.(*ChannelQueue) - queue.delayedStarter = delayedStarter{ - cfg: levelCfg, - underlying: LevelQueueType, - timeout: config.Timeout, - maxAttempts: config.MaxAttempts, - name: config.Name, - } - _ = GetManager().Add(queue, PersistableChannelQueueType, config, exemplar) - return queue, nil -} - -// Name returns the name of this queue -func (q *PersistableChannelQueue) Name() string { - return q.delayedStarter.name -} - -// Push will push the indexer data to queue -func (q *PersistableChannelQueue) Push(data Data) error { - select { - case <-q.closed: - return q.internal.Push(data) - default: - return q.channelQueue.Push(data) - } -} - -// PushBack will push the indexer data to queue -func (q *PersistableChannelQueue) PushBack(data Data) error { - select { - case <-q.closed: - if pbr, ok := q.internal.(PushBackable); ok { - return pbr.PushBack(data) - } - return q.internal.Push(data) - default: - return q.channelQueue.Push(data) - } -} - -// Run starts to run the queue -func (q *PersistableChannelQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.channelQueue.baseCtx) - log.Debug("PersistableChannelQueue: %s Starting", q.delayedStarter.name) - _ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0) - - q.lock.Lock() - if q.internal == nil { - err := q.setInternal(atShutdown, q.channelQueue.handle, q.channelQueue.exemplar) - q.lock.Unlock() - if err != nil { - log.Fatal("Unable to create internal queue for %s Error: %v", q.Name(), err) - return - } - } else { - q.lock.Unlock() - } - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - - if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.terminateCtx) != 0 { - // Just run the level queue - we shut it down once it's flushed - go q.internal.Run(func(_ func()) {}, func(_ func()) {}) - go func() { - for !lq.IsEmpty() { - _ = lq.Flush(0) - select { - case <-time.After(100 * time.Millisecond): - case <-lq.shutdownCtx.Done(): - if lq.byteFIFO.Len(lq.terminateCtx) > 0 { - log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name()) - } - return - } - } - log.Debug("LevelQueue: %s flushed so shutting down", q.internal.(*LevelQueue).Name()) - q.internal.(*LevelQueue).Shutdown() - GetManager().Remove(q.internal.(*LevelQueue).qid) - }() - } else { - log.Debug("PersistableChannelQueue: %s Skipping running the empty level queue", q.delayedStarter.name) - q.internal.(*LevelQueue).Shutdown() - GetManager().Remove(q.internal.(*LevelQueue).qid) - } -} - -// Flush flushes the queue and blocks till the queue is empty -func (q *PersistableChannelQueue) Flush(timeout time.Duration) error { - var ctx context.Context - var cancel context.CancelFunc - if timeout > 0 { - ctx, cancel = context.WithTimeout(context.Background(), timeout) - } else { - ctx, cancel = context.WithCancel(context.Background()) - } - defer cancel() - return q.FlushWithContext(ctx) -} - -// FlushWithContext flushes the queue and blocks till the queue is empty -func (q *PersistableChannelQueue) FlushWithContext(ctx context.Context) error { - errChan := make(chan error, 1) - go func() { - errChan <- q.channelQueue.FlushWithContext(ctx) - }() - go func() { - q.lock.Lock() - if q.internal == nil { - q.lock.Unlock() - errChan <- fmt.Errorf("not ready to flush internal queue %s yet", q.Name()) - return - } - q.lock.Unlock() - errChan <- q.internal.FlushWithContext(ctx) - }() - err1 := <-errChan - err2 := <-errChan - - if err1 != nil { - return err1 - } - return err2 -} - -// IsEmpty checks if a queue is empty -func (q *PersistableChannelQueue) IsEmpty() bool { - if !q.channelQueue.IsEmpty() { - return false - } - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return false - } - return q.internal.IsEmpty() -} - -// IsPaused returns if the pool is paused -func (q *PersistableChannelQueue) IsPaused() bool { - return q.channelQueue.IsPaused() -} - -// IsPausedIsResumed returns if the pool is paused and a channel that is closed when it is resumed -func (q *PersistableChannelQueue) IsPausedIsResumed() (<-chan struct{}, <-chan struct{}) { - return q.channelQueue.IsPausedIsResumed() -} - -// Pause pauses the WorkerPool -func (q *PersistableChannelQueue) Pause() { - q.channelQueue.Pause() - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return - } - - pausable, ok := q.internal.(Pausable) - if !ok { - return - } - pausable.Pause() -} - -// Resume resumes the WorkerPool -func (q *PersistableChannelQueue) Resume() { - q.channelQueue.Resume() - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return - } - - pausable, ok := q.internal.(Pausable) - if !ok { - return - } - pausable.Resume() -} - -// Shutdown processing this queue -func (q *PersistableChannelQueue) Shutdown() { - log.Trace("PersistableChannelQueue: %s Shutting down", q.delayedStarter.name) - q.lock.Lock() - - select { - case <-q.closed: - q.lock.Unlock() - return - default: - } - q.channelQueue.Shutdown() - if q.internal != nil { - q.internal.(*LevelQueue).Shutdown() - } - close(q.closed) - q.lock.Unlock() - - log.Trace("PersistableChannelQueue: %s Cancelling pools", q.delayedStarter.name) - q.channelQueue.baseCtxCancel() - q.internal.(*LevelQueue).baseCtxCancel() - log.Trace("PersistableChannelQueue: %s Waiting til done", q.delayedStarter.name) - q.channelQueue.Wait() - q.internal.(*LevelQueue).Wait() - // Redirect all remaining data in the chan to the internal channel - log.Trace("PersistableChannelQueue: %s Redirecting remaining data", q.delayedStarter.name) - close(q.channelQueue.dataChan) - countOK, countLost := 0, 0 - for data := range q.channelQueue.dataChan { - err := q.internal.Push(data) - if err != nil { - log.Error("PersistableChannelQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err) - countLost++ - } else { - countOK++ - } - atomic.AddInt64(&q.channelQueue.numInQueue, -1) - } - if countLost > 0 { - log.Warn("PersistableChannelQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost) - } else if countOK > 0 { - log.Warn("PersistableChannelQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK) - } - log.Trace("PersistableChannelQueue: %s Done Redirecting remaining data", q.delayedStarter.name) - - log.Debug("PersistableChannelQueue: %s Shutdown", q.delayedStarter.name) -} - -// Terminate this queue and close the queue -func (q *PersistableChannelQueue) Terminate() { - log.Trace("PersistableChannelQueue: %s Terminating", q.delayedStarter.name) - q.Shutdown() - q.lock.Lock() - defer q.lock.Unlock() - q.channelQueue.Terminate() - if q.internal != nil { - q.internal.(*LevelQueue).Terminate() - } - log.Debug("PersistableChannelQueue: %s Terminated", q.delayedStarter.name) -} - -func init() { - queuesMap[PersistableChannelQueueType] = NewPersistableChannelQueue -} diff --git a/modules/queue/queue_disk_channel_test.go b/modules/queue/queue_disk_channel_test.go deleted file mode 100644 index 4f14a5d79df..00000000000 --- a/modules/queue/queue_disk_channel_test.go +++ /dev/null @@ -1,544 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/log" - - "github.com/stretchr/testify/assert" -) - -func TestPersistableChannelQueue(t *testing.T) { - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - for _, datum := range data { - if datum == nil { - continue - } - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - lock := sync.Mutex{} - queueShutdown := []func(){} - queueTerminate := []func(){} - - tmpDir := t.TempDir() - - queue, err := NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ - DataDir: tmpDir, - BatchLength: 2, - QueueLength: 20, - Workers: 1, - BoostWorkers: 0, - MaxWorkers: 10, - Name: "test-queue", - }, &testData{}) - assert.NoError(t, err) - - readyForShutdown := make(chan struct{}) - readyForTerminate := make(chan struct{}) - - go queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-readyForShutdown: - default: - close(readyForShutdown) - } - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-readyForTerminate: - default: - close(readyForTerminate) - } - queueTerminate = append(queueTerminate, terminate) - }) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - err = queue.Push(&test1) - assert.NoError(t, err) - go func() { - err := queue.Push(&test2) - assert.NoError(t, err) - }() - - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - result2 := <-handleChan - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - // test1 is a testData not a *testData so will be rejected - err = queue.Push(test1) - assert.Error(t, err) - - <-readyForShutdown - // Now shutdown the queue - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - - // Wait til it is closed - <-queue.(*PersistableChannelQueue).closed - - err = queue.Push(&test1) - assert.NoError(t, err) - err = queue.Push(&test2) - assert.NoError(t, err) - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - default: - } - - // terminate the queue - <-readyForTerminate - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - default: - } - - // Reopen queue - queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ - DataDir: tmpDir, - BatchLength: 2, - QueueLength: 20, - Workers: 1, - BoostWorkers: 0, - MaxWorkers: 10, - Name: "test-queue", - }, &testData{}) - assert.NoError(t, err) - - readyForShutdown = make(chan struct{}) - readyForTerminate = make(chan struct{}) - - go queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-readyForShutdown: - default: - close(readyForShutdown) - } - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-readyForTerminate: - default: - close(readyForTerminate) - } - queueTerminate = append(queueTerminate, terminate) - }) - - result3 := <-handleChan - assert.Equal(t, test1.TestString, result3.TestString) - assert.Equal(t, test1.TestInt, result3.TestInt) - - result4 := <-handleChan - assert.Equal(t, test2.TestString, result4.TestString) - assert.Equal(t, test2.TestInt, result4.TestInt) - - <-readyForShutdown - lock.Lock() - callbacks = make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - <-readyForTerminate - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - lock.Unlock() - for _, callback := range callbacks { - callback() - } -} - -func TestPersistableChannelQueue_Pause(t *testing.T) { - lock := sync.Mutex{} - var queue Queue - var err error - pushBack := false - - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - lock.Lock() - if pushBack { - if pausable, ok := queue.(Pausable); ok { - log.Info("pausing") - pausable.Pause() - } - lock.Unlock() - return data - } - lock.Unlock() - - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - queueShutdown := []func(){} - queueTerminate := []func(){} - terminated := make(chan struct{}) - - tmpDir := t.TempDir() - - queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ - DataDir: tmpDir, - BatchLength: 2, - QueueLength: 20, - Workers: 1, - BoostWorkers: 0, - MaxWorkers: 10, - Name: "test-queue", - }, &testData{}) - assert.NoError(t, err) - - go func() { - queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) - close(terminated) - }() - - // Shutdown and Terminate in defer - defer func() { - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - lock.Lock() - log.Info("Finally terminating") - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - }() - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - err = queue.Push(&test1) - assert.NoError(t, err) - - pausable, ok := queue.(Pausable) - if !assert.True(t, ok) { - return - } - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - pausable.Pause() - paused, _ := pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not paused") - return - } - - queue.Push(&test2) - - var result2 *testData - select { - case result2 = <-handleChan: - assert.Fail(t, "handler chan should be empty") - case <-time.After(100 * time.Millisecond): - } - - assert.Nil(t, result2) - - pausable.Resume() - _, resumed := pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - return - } - - select { - case result2 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test2") - } - - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - // Set pushBack to so that the next handle will result in a Pause - lock.Lock() - pushBack = true - lock.Unlock() - - // Ensure that we're still resumed - _, resumed = pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not resumed") - return - } - - // push test1 - queue.Push(&test1) - - // Now as this is handled it should pause - paused, _ = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-handleChan: - assert.Fail(t, "handler chan should not contain test1") - return - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "queue should be paused") - return - } - - lock.Lock() - pushBack = false - lock.Unlock() - - pausable.Resume() - - _, resumed = pausable.IsPausedIsResumed() - select { - case <-resumed: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - return - } - - select { - case result1 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test1") - return - } - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - queueShutdown = queueShutdown[:0] - lock.Unlock() - // Now shutdown the queue - for _, callback := range callbacks { - callback() - } - - // Wait til it is closed - select { - case <-queue.(*PersistableChannelQueue).closed: - case <-time.After(5 * time.Second): - assert.Fail(t, "queue should close") - return - } - - err = queue.Push(&test1) - assert.NoError(t, err) - err = queue.Push(&test2) - assert.NoError(t, err) - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - return - default: - } - - // terminate the queue - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - queueShutdown = queueTerminate[:0] - lock.Unlock() - for _, callback := range callbacks { - callback() - } - - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - return - case <-terminated: - case <-time.After(10 * time.Second): - assert.Fail(t, "Queue should have terminated") - return - } - - lock.Lock() - pushBack = true - lock.Unlock() - - // Reopen queue - terminated = make(chan struct{}) - queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ - DataDir: tmpDir, - BatchLength: 1, - QueueLength: 20, - Workers: 1, - BoostWorkers: 0, - MaxWorkers: 10, - Name: "test-queue", - }, &testData{}) - assert.NoError(t, err) - pausable, ok = queue.(Pausable) - if !assert.True(t, ok) { - return - } - - paused, _ = pausable.IsPausedIsResumed() - - go func() { - queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) - close(terminated) - }() - - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - return - case <-paused: - } - - paused, _ = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "Queue is not paused") - return - } - - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - return - default: - } - - lock.Lock() - pushBack = false - lock.Unlock() - - pausable.Resume() - _, resumed = pausable.IsPausedIsResumed() - select { - case <-resumed: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - return - } - - var result3, result4 *testData - - select { - case result3 = <-handleChan: - case <-time.After(1 * time.Second): - assert.Fail(t, "Handler processing should have resumed") - return - } - select { - case result4 = <-handleChan: - case <-time.After(1 * time.Second): - assert.Fail(t, "Handler processing should have resumed") - return - } - if result4.TestString == test1.TestString { - result3, result4 = result4, result3 - } - assert.Equal(t, test1.TestString, result3.TestString) - assert.Equal(t, test1.TestInt, result3.TestInt) - - assert.Equal(t, test2.TestString, result4.TestString) - assert.Equal(t, test2.TestInt, result4.TestInt) - - lock.Lock() - callbacks = make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - queueShutdown = queueShutdown[:0] - lock.Unlock() - // Now shutdown the queue - for _, callback := range callbacks { - callback() - } - - // terminate the queue - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - queueShutdown = queueTerminate[:0] - lock.Unlock() - for _, callback := range callbacks { - callback() - } - - select { - case <-time.After(10 * time.Second): - assert.Fail(t, "Queue should have terminated") - return - case <-terminated: - } -} diff --git a/modules/queue/queue_disk_test.go b/modules/queue/queue_disk_test.go deleted file mode 100644 index 8f83abf42c0..00000000000 --- a/modules/queue/queue_disk_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestLevelQueue(t *testing.T) { - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - assert.True(t, len(data) == 2) - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - var lock sync.Mutex - queueShutdown := []func(){} - queueTerminate := []func(){} - - tmpDir := t.TempDir() - - queue, err := NewLevelQueue(handle, LevelQueueConfiguration{ - ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 2, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 5, - MaxWorkers: 10, - }, - Workers: 1, - }, - DataDir: tmpDir, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(func(shutdown func()) { - lock.Lock() - queueShutdown = append(queueShutdown, shutdown) - lock.Unlock() - }, func(terminate func()) { - lock.Lock() - queueTerminate = append(queueTerminate, terminate) - lock.Unlock() - }) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - err = queue.Push(&test1) - assert.NoError(t, err) - go func() { - err := queue.Push(&test2) - assert.NoError(t, err) - }() - - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - result2 := <-handleChan - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) - - lock.Lock() - for _, callback := range queueShutdown { - callback() - } - lock.Unlock() - - time.Sleep(200 * time.Millisecond) - err = queue.Push(&test1) - assert.NoError(t, err) - err = queue.Push(&test2) - assert.NoError(t, err) - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - default: - } - lock.Lock() - for _, callback := range queueTerminate { - callback() - } - lock.Unlock() - - // Reopen queue - queue, err = NewWrappedQueue(handle, - WrappedQueueConfiguration{ - Underlying: LevelQueueType, - Config: LevelQueueConfiguration{ - ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 2, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 5, - MaxWorkers: 10, - }, - Workers: 1, - }, - DataDir: tmpDir, - }, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(func(shutdown func()) { - lock.Lock() - queueShutdown = append(queueShutdown, shutdown) - lock.Unlock() - }, func(terminate func()) { - lock.Lock() - queueTerminate = append(queueTerminate, terminate) - lock.Unlock() - }) - - result3 := <-handleChan - assert.Equal(t, test1.TestString, result3.TestString) - assert.Equal(t, test1.TestInt, result3.TestInt) - - result4 := <-handleChan - assert.Equal(t, test2.TestString, result4.TestString) - assert.Equal(t, test2.TestInt, result4.TestInt) - - lock.Lock() - for _, callback := range queueShutdown { - callback() - } - for _, callback := range queueTerminate { - callback() - } - lock.Unlock() -} diff --git a/modules/queue/queue_redis.go b/modules/queue/queue_redis.go deleted file mode 100644 index f8842fea9fa..00000000000 --- a/modules/queue/queue_redis.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - - "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/nosql" - - "github.com/redis/go-redis/v9" -) - -// RedisQueueType is the type for redis queue -const RedisQueueType Type = "redis" - -// RedisQueueConfiguration is the configuration for the redis queue -type RedisQueueConfiguration struct { - ByteFIFOQueueConfiguration - RedisByteFIFOConfiguration -} - -// RedisQueue redis queue -type RedisQueue struct { - *ByteFIFOQueue -} - -// NewRedisQueue creates single redis or cluster redis queue -func NewRedisQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(RedisQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(RedisQueueConfiguration) - - byteFIFO, err := NewRedisByteFIFO(config.RedisByteFIFOConfiguration) - if err != nil { - return nil, err - } - - byteFIFOQueue, err := NewByteFIFOQueue(RedisQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) - if err != nil { - return nil, err - } - - queue := &RedisQueue{ - ByteFIFOQueue: byteFIFOQueue, - } - - queue.qid = GetManager().Add(queue, RedisQueueType, config, exemplar) - - return queue, nil -} - -type redisClient interface { - RPush(ctx context.Context, key string, args ...interface{}) *redis.IntCmd - LPush(ctx context.Context, key string, args ...interface{}) *redis.IntCmd - LPop(ctx context.Context, key string) *redis.StringCmd - LLen(ctx context.Context, key string) *redis.IntCmd - SAdd(ctx context.Context, key string, members ...interface{}) *redis.IntCmd - SRem(ctx context.Context, key string, members ...interface{}) *redis.IntCmd - SIsMember(ctx context.Context, key string, member interface{}) *redis.BoolCmd - Ping(ctx context.Context) *redis.StatusCmd - Close() error -} - -var _ ByteFIFO = &RedisByteFIFO{} - -// RedisByteFIFO represents a ByteFIFO formed from a redisClient -type RedisByteFIFO struct { - client redisClient - - queueName string -} - -// RedisByteFIFOConfiguration is the configuration for the RedisByteFIFO -type RedisByteFIFOConfiguration struct { - ConnectionString string - QueueName string -} - -// NewRedisByteFIFO creates a ByteFIFO formed from a redisClient -func NewRedisByteFIFO(config RedisByteFIFOConfiguration) (*RedisByteFIFO, error) { - fifo := &RedisByteFIFO{ - queueName: config.QueueName, - } - fifo.client = nosql.GetManager().GetRedisClient(config.ConnectionString) - if err := fifo.client.Ping(graceful.GetManager().ShutdownContext()).Err(); err != nil { - return nil, err - } - return fifo, nil -} - -// PushFunc pushes data to the end of the fifo and calls the callback if it is added -func (fifo *RedisByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - if fn != nil { - if err := fn(); err != nil { - return err - } - } - return fifo.client.RPush(ctx, fifo.queueName, data).Err() -} - -// PushBack pushes data to the top of the fifo -func (fifo *RedisByteFIFO) PushBack(ctx context.Context, data []byte) error { - return fifo.client.LPush(ctx, fifo.queueName, data).Err() -} - -// Pop pops data from the start of the fifo -func (fifo *RedisByteFIFO) Pop(ctx context.Context) ([]byte, error) { - data, err := fifo.client.LPop(ctx, fifo.queueName).Bytes() - if err == nil || err == redis.Nil { - return data, nil - } - return data, err -} - -// Close this fifo -func (fifo *RedisByteFIFO) Close() error { - return fifo.client.Close() -} - -// Len returns the length of the fifo -func (fifo *RedisByteFIFO) Len(ctx context.Context) int64 { - val, err := fifo.client.LLen(ctx, fifo.queueName).Result() - if err != nil { - log.Error("Error whilst getting length of redis queue %s: Error: %v", fifo.queueName, err) - return -1 - } - return val -} - -func init() { - queuesMap[RedisQueueType] = NewRedisQueue -} diff --git a/modules/queue/queue_test.go b/modules/queue/queue_test.go deleted file mode 100644 index 42d34c806ce..00000000000 --- a/modules/queue/queue_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "testing" - - "code.gitea.io/gitea/modules/json" - - "github.com/stretchr/testify/assert" -) - -type testData struct { - TestString string - TestInt int -} - -func TestToConfig(t *testing.T) { - cfg := testData{ - TestString: "Config", - TestInt: 10, - } - exemplar := testData{} - - cfg2I, err := toConfig(exemplar, cfg) - assert.NoError(t, err) - cfg2, ok := (cfg2I).(testData) - assert.True(t, ok) - assert.NotEqual(t, cfg2, exemplar) - assert.Equal(t, &cfg, &cfg2) - cfgString, err := json.Marshal(cfg) - assert.NoError(t, err) - - cfg3I, err := toConfig(exemplar, cfgString) - assert.NoError(t, err) - cfg3, ok := (cfg3I).(testData) - assert.True(t, ok) - assert.Equal(t, cfg.TestString, cfg3.TestString) - assert.Equal(t, cfg.TestInt, cfg3.TestInt) - assert.NotEqual(t, cfg3, exemplar) -} diff --git a/modules/queue/queue_wrapped.go b/modules/queue/queue_wrapped.go deleted file mode 100644 index 84d6dd98a5a..00000000000 --- a/modules/queue/queue_wrapped.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "sync" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" -) - -// WrappedQueueType is the type for a wrapped delayed starting queue -const WrappedQueueType Type = "wrapped" - -// WrappedQueueConfiguration is the configuration for a WrappedQueue -type WrappedQueueConfiguration struct { - Underlying Type - Timeout time.Duration - MaxAttempts int - Config interface{} - QueueLength int - Name string -} - -type delayedStarter struct { - internal Queue - underlying Type - cfg interface{} - timeout time.Duration - maxAttempts int - name string -} - -// setInternal must be called with the lock locked. -func (q *delayedStarter) setInternal(atShutdown func(func()), handle HandlerFunc, exemplar interface{}) error { - var ctx context.Context - var cancel context.CancelFunc - if q.timeout > 0 { - ctx, cancel = context.WithTimeout(context.Background(), q.timeout) - } else { - ctx, cancel = context.WithCancel(context.Background()) - } - - defer cancel() - // Ensure we also stop at shutdown - atShutdown(cancel) - - i := 1 - for q.internal == nil { - select { - case <-ctx.Done(): - cfg := q.cfg - if s, ok := cfg.([]byte); ok { - cfg = string(s) - } - return fmt.Errorf("timedout creating queue %v with cfg %#v in %s", q.underlying, cfg, q.name) - default: - queue, err := NewQueue(q.underlying, handle, q.cfg, exemplar) - if err == nil { - q.internal = queue - break - } - if err.Error() != "resource temporarily unavailable" { - if bs, ok := q.cfg.([]byte); ok { - log.Warn("[Attempt: %d] Failed to create queue: %v for %s cfg: %s error: %v", i, q.underlying, q.name, string(bs), err) - } else { - log.Warn("[Attempt: %d] Failed to create queue: %v for %s cfg: %#v error: %v", i, q.underlying, q.name, q.cfg, err) - } - } - i++ - if q.maxAttempts > 0 && i > q.maxAttempts { - if bs, ok := q.cfg.([]byte); ok { - return fmt.Errorf("unable to create queue %v for %s with cfg %s by max attempts: error: %w", q.underlying, q.name, string(bs), err) - } - return fmt.Errorf("unable to create queue %v for %s with cfg %#v by max attempts: error: %w", q.underlying, q.name, q.cfg, err) - } - sleepTime := 100 * time.Millisecond - if q.timeout > 0 && q.maxAttempts > 0 { - sleepTime = (q.timeout - 200*time.Millisecond) / time.Duration(q.maxAttempts) - } - t := time.NewTimer(sleepTime) - select { - case <-ctx.Done(): - util.StopTimer(t) - case <-t.C: - } - } - } - return nil -} - -// WrappedQueue wraps a delayed starting queue -type WrappedQueue struct { - delayedStarter - lock sync.Mutex - handle HandlerFunc - exemplar interface{} - channel chan Data - numInQueue int64 -} - -// NewWrappedQueue will attempt to create a queue of the provided type, -// but if there is a problem creating this queue it will instead create -// a WrappedQueue with delayed startup of the queue instead and a -// channel which will be redirected to the queue -func NewWrappedQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(WrappedQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(WrappedQueueConfiguration) - - queue, err := NewQueue(config.Underlying, handle, config.Config, exemplar) - if err == nil { - // Just return the queue there is no need to wrap - return queue, nil - } - if IsErrInvalidConfiguration(err) { - // Retrying ain't gonna make this any better... - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - queue = &WrappedQueue{ - handle: handle, - channel: make(chan Data, config.QueueLength), - exemplar: exemplar, - delayedStarter: delayedStarter{ - cfg: config.Config, - underlying: config.Underlying, - timeout: config.Timeout, - maxAttempts: config.MaxAttempts, - name: config.Name, - }, - } - _ = GetManager().Add(queue, WrappedQueueType, config, exemplar) - return queue, nil -} - -// Name returns the name of the queue -func (q *WrappedQueue) Name() string { - return q.name + "-wrapper" -} - -// Push will push the data to the internal channel checking it against the exemplar -func (q *WrappedQueue) Push(data Data) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - atomic.AddInt64(&q.numInQueue, 1) - q.channel <- data - return nil -} - -func (q *WrappedQueue) flushInternalWithContext(ctx context.Context) error { - q.lock.Lock() - if q.internal == nil { - q.lock.Unlock() - return fmt.Errorf("not ready to flush wrapped queue %s yet", q.Name()) - } - q.lock.Unlock() - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - return q.internal.FlushWithContext(ctx) -} - -// Flush flushes the queue and blocks till the queue is empty -func (q *WrappedQueue) Flush(timeout time.Duration) error { - var ctx context.Context - var cancel context.CancelFunc - if timeout > 0 { - ctx, cancel = context.WithTimeout(context.Background(), timeout) - } else { - ctx, cancel = context.WithCancel(context.Background()) - } - defer cancel() - return q.FlushWithContext(ctx) -} - -// FlushWithContext implements the final part of Flushable -func (q *WrappedQueue) FlushWithContext(ctx context.Context) error { - log.Trace("WrappedQueue: %s FlushWithContext", q.Name()) - errChan := make(chan error, 1) - go func() { - errChan <- q.flushInternalWithContext(ctx) - close(errChan) - }() - - select { - case err := <-errChan: - return err - case <-ctx.Done(): - go func() { - <-errChan - }() - return ctx.Err() - } -} - -// IsEmpty checks whether the queue is empty -func (q *WrappedQueue) IsEmpty() bool { - if atomic.LoadInt64(&q.numInQueue) != 0 { - return false - } - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return false - } - return q.internal.IsEmpty() -} - -// Run starts to run the queue and attempts to create the internal queue -func (q *WrappedQueue) Run(atShutdown, atTerminate func(func())) { - log.Debug("WrappedQueue: %s Starting", q.name) - q.lock.Lock() - if q.internal == nil { - err := q.setInternal(atShutdown, q.handle, q.exemplar) - q.lock.Unlock() - if err != nil { - log.Fatal("Unable to set the internal queue for %s Error: %v", q.Name(), err) - return - } - go func() { - for data := range q.channel { - _ = q.internal.Push(data) - atomic.AddInt64(&q.numInQueue, -1) - } - }() - } else { - q.lock.Unlock() - } - - q.internal.Run(atShutdown, atTerminate) - log.Trace("WrappedQueue: %s Done", q.name) -} - -// Shutdown this queue and stop processing -func (q *WrappedQueue) Shutdown() { - log.Trace("WrappedQueue: %s Shutting down", q.name) - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return - } - if shutdownable, ok := q.internal.(Shutdownable); ok { - shutdownable.Shutdown() - } - log.Debug("WrappedQueue: %s Shutdown", q.name) -} - -// Terminate this queue and close the queue -func (q *WrappedQueue) Terminate() { - log.Trace("WrappedQueue: %s Terminating", q.name) - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return - } - if shutdownable, ok := q.internal.(Shutdownable); ok { - shutdownable.Terminate() - } - log.Debug("WrappedQueue: %s Terminated", q.name) -} - -// IsPaused will return if the pool or queue is paused -func (q *WrappedQueue) IsPaused() bool { - q.lock.Lock() - defer q.lock.Unlock() - pausable, ok := q.internal.(Pausable) - return ok && pausable.IsPaused() -} - -// Pause will pause the pool or queue -func (q *WrappedQueue) Pause() { - q.lock.Lock() - defer q.lock.Unlock() - if pausable, ok := q.internal.(Pausable); ok { - pausable.Pause() - } -} - -// Resume will resume the pool or queue -func (q *WrappedQueue) Resume() { - q.lock.Lock() - defer q.lock.Unlock() - if pausable, ok := q.internal.(Pausable); ok { - pausable.Resume() - } -} - -// IsPausedIsResumed will return a bool indicating if the pool or queue is paused and a channel that will be closed when it is resumed -func (q *WrappedQueue) IsPausedIsResumed() (paused, resumed <-chan struct{}) { - q.lock.Lock() - defer q.lock.Unlock() - if pausable, ok := q.internal.(Pausable); ok { - return pausable.IsPausedIsResumed() - } - return context.Background().Done(), closedChan -} - -var closedChan chan struct{} - -func init() { - queuesMap[WrappedQueueType] = NewWrappedQueue - closedChan = make(chan struct{}) - close(closedChan) -} diff --git a/modules/queue/setting.go b/modules/queue/setting.go deleted file mode 100644 index 1e5259fcfba..00000000000 --- a/modules/queue/setting.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "fmt" - "strings" - - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -func validType(t string) (Type, error) { - if len(t) == 0 { - return PersistableChannelQueueType, nil - } - for _, typ := range RegisteredTypes() { - if t == string(typ) { - return typ, nil - } - } - return PersistableChannelQueueType, fmt.Errorf("unknown queue type: %s defaulting to %s", t, string(PersistableChannelQueueType)) -} - -func getQueueSettings(name string) (setting.QueueSettings, []byte) { - q := setting.GetQueueSettings(name) - cfg, err := json.Marshal(q) - if err != nil { - log.Error("Unable to marshall generic options: %v Error: %v", q, err) - log.Error("Unable to create queue for %s", name, err) - return q, []byte{} - } - return q, cfg -} - -// CreateQueue for name with provided handler and exemplar -func CreateQueue(name string, handle HandlerFunc, exemplar interface{}) Queue { - q, cfg := getQueueSettings(name) - if len(cfg) == 0 { - return nil - } - - typ, err := validType(q.Type) - if err != nil { - log.Error("Invalid type %s provided for queue named %s defaulting to %s", q.Type, name, string(typ)) - } - - returnable, err := NewQueue(typ, handle, cfg, exemplar) - if q.WrapIfNecessary && err != nil { - log.Warn("Unable to create queue for %s: %v", name, err) - log.Warn("Attempting to create wrapped queue") - returnable, err = NewQueue(WrappedQueueType, handle, WrappedQueueConfiguration{ - Underlying: typ, - Timeout: q.Timeout, - MaxAttempts: q.MaxAttempts, - Config: cfg, - QueueLength: q.QueueLength, - Name: name, - }, exemplar) - } - if err != nil { - log.Error("Unable to create queue for %s: %v", name, err) - return nil - } - - // Sanity check configuration - if q.Workers == 0 && (q.BoostTimeout == 0 || q.BoostWorkers == 0 || q.MaxWorkers == 0) { - log.Warn("Queue: %s is configured to be non-scaling and have no workers\n - this configuration is likely incorrect and could cause Gitea to block", q.Name) - if pausable, ok := returnable.(Pausable); ok { - log.Warn("Queue: %s is being paused to prevent data-loss, add workers manually and unpause.", q.Name) - pausable.Pause() - } - } - - return returnable -} - -// CreateUniqueQueue for name with provided handler and exemplar -func CreateUniqueQueue(name string, handle HandlerFunc, exemplar interface{}) UniqueQueue { - q, cfg := getQueueSettings(name) - if len(cfg) == 0 { - return nil - } - - if len(q.Type) > 0 && q.Type != "dummy" && q.Type != "immediate" && !strings.HasPrefix(q.Type, "unique-") { - q.Type = "unique-" + q.Type - } - - typ, err := validType(q.Type) - if err != nil || typ == PersistableChannelQueueType { - typ = PersistableChannelUniqueQueueType - if err != nil { - log.Error("Invalid type %s provided for queue named %s defaulting to %s", q.Type, name, string(typ)) - } - } - - returnable, err := NewQueue(typ, handle, cfg, exemplar) - if q.WrapIfNecessary && err != nil { - log.Warn("Unable to create unique queue for %s: %v", name, err) - log.Warn("Attempting to create wrapped queue") - returnable, err = NewQueue(WrappedUniqueQueueType, handle, WrappedUniqueQueueConfiguration{ - Underlying: typ, - Timeout: q.Timeout, - MaxAttempts: q.MaxAttempts, - Config: cfg, - QueueLength: q.QueueLength, - }, exemplar) - } - if err != nil { - log.Error("Unable to create unique queue for %s: %v", name, err) - return nil - } - - // Sanity check configuration - if q.Workers == 0 && (q.BoostTimeout == 0 || q.BoostWorkers == 0 || q.MaxWorkers == 0) { - log.Warn("Queue: %s is configured to be non-scaling and have no workers\n - this configuration is likely incorrect and could cause Gitea to block", q.Name) - if pausable, ok := returnable.(Pausable); ok { - log.Warn("Queue: %s is being paused to prevent data-loss, add workers manually and unpause.", q.Name) - pausable.Pause() - } - } - - return returnable.(UniqueQueue) -} diff --git a/modules/queue/testhelper.go b/modules/queue/testhelper.go new file mode 100644 index 00000000000..edfa438b1a8 --- /dev/null +++ b/modules/queue/testhelper.go @@ -0,0 +1,40 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "fmt" + "sync" +) + +// testStateRecorder is used to record state changes for testing, to help debug async behaviors +type testStateRecorder struct { + records []string + mu sync.Mutex +} + +var testRecorder = &testStateRecorder{} + +func (t *testStateRecorder) Record(format string, args ...any) { + t.mu.Lock() + t.records = append(t.records, fmt.Sprintf(format, args...)) + if len(t.records) > 1000 { + t.records = t.records[len(t.records)-1000:] + } + t.mu.Unlock() +} + +func (t *testStateRecorder) Records() []string { + t.mu.Lock() + r := make([]string, len(t.records)) + copy(r, t.records) + t.mu.Unlock() + return r +} + +func (t *testStateRecorder) Reset() { + t.mu.Lock() + t.records = nil + t.mu.Unlock() +} diff --git a/modules/queue/unique_queue.go b/modules/queue/unique_queue.go deleted file mode 100644 index 8f8215c71d8..00000000000 --- a/modules/queue/unique_queue.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "fmt" -) - -// UniqueQueue defines a queue which guarantees only one instance of same -// data is in the queue. Instances with same identity will be -// discarded if there is already one in the line. -// -// This queue is particularly useful for preventing duplicated task -// of same purpose - please note that this does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -// -// Users of this queue should be careful to push only the identifier of the -// data -type UniqueQueue interface { - Queue - PushFunc(Data, func() error) error - Has(Data) (bool, error) -} - -// ErrAlreadyInQueue is returned when trying to push data to the queue that is already in the queue -var ErrAlreadyInQueue = fmt.Errorf("already in queue") diff --git a/modules/queue/unique_queue_channel.go b/modules/queue/unique_queue_channel.go deleted file mode 100644 index 62c051aa393..00000000000 --- a/modules/queue/unique_queue_channel.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "time" - - "code.gitea.io/gitea/modules/container" - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" -) - -// ChannelUniqueQueueType is the type for channel queue -const ChannelUniqueQueueType Type = "unique-channel" - -// ChannelUniqueQueueConfiguration is the configuration for a ChannelUniqueQueue -type ChannelUniqueQueueConfiguration ChannelQueueConfiguration - -// ChannelUniqueQueue implements UniqueQueue -// -// It is basically a thin wrapper around a WorkerPool but keeps a store of -// what has been pushed within a table. -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -type ChannelUniqueQueue struct { - *WorkerPool - lock sync.Mutex - table container.Set[string] - shutdownCtx context.Context - shutdownCtxCancel context.CancelFunc - terminateCtx context.Context - terminateCtxCancel context.CancelFunc - exemplar interface{} - workers int - name string -} - -// NewChannelUniqueQueue create a memory channel queue -func NewChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(ChannelUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(ChannelUniqueQueueConfiguration) - if config.BatchLength == 0 { - config.BatchLength = 1 - } - - terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) - shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) - - queue := &ChannelUniqueQueue{ - table: make(container.Set[string]), - shutdownCtx: shutdownCtx, - shutdownCtxCancel: shutdownCtxCancel, - terminateCtx: terminateCtx, - terminateCtxCancel: terminateCtxCancel, - exemplar: exemplar, - workers: config.Workers, - name: config.Name, - } - queue.WorkerPool = NewWorkerPool(func(data ...Data) (unhandled []Data) { - for _, datum := range data { - // No error is possible here because PushFunc ensures that this can be marshalled - bs, _ := json.Marshal(datum) - - queue.lock.Lock() - queue.table.Remove(string(bs)) - queue.lock.Unlock() - - if u := handle(datum); u != nil { - if queue.IsPaused() { - // We can only pushback to the channel if we're paused. - go func() { - if err := queue.Push(u[0]); err != nil { - log.Error("Unable to push back to queue %d. Error: %v", queue.qid, err) - } - }() - } else { - unhandled = append(unhandled, u...) - } - } - } - return unhandled - }, config.WorkerPoolConfiguration) - - queue.qid = GetManager().Add(queue, ChannelUniqueQueueType, config, exemplar) - return queue, nil -} - -// Run starts to run the queue -func (q *ChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.baseCtx) - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - log.Debug("ChannelUniqueQueue: %s Starting", q.name) - _ = q.AddWorkers(q.workers, 0) -} - -// Push will push data into the queue if the data is not already in the queue -func (q *ChannelUniqueQueue) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushFunc will push data into the queue -func (q *ChannelUniqueQueue) PushFunc(data Data, fn func() error) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in queue: %s", data, q.exemplar, q.name) - } - - bs, err := json.Marshal(data) - if err != nil { - return err - } - q.lock.Lock() - locked := true - defer func() { - if locked { - q.lock.Unlock() - } - }() - if !q.table.Add(string(bs)) { - return ErrAlreadyInQueue - } - // FIXME: We probably need to implement some sort of limit here - // If the downstream queue blocks this table will grow without limit - if fn != nil { - err := fn() - if err != nil { - q.table.Remove(string(bs)) - return err - } - } - locked = false - q.lock.Unlock() - q.WorkerPool.Push(data) - return nil -} - -// Has checks if the data is in the queue -func (q *ChannelUniqueQueue) Has(data Data) (bool, error) { - bs, err := json.Marshal(data) - if err != nil { - return false, err - } - - q.lock.Lock() - defer q.lock.Unlock() - return q.table.Contains(string(bs)), nil -} - -// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager -func (q *ChannelUniqueQueue) Flush(timeout time.Duration) error { - if q.IsPaused() { - return nil - } - ctx, cancel := q.commonRegisterWorkers(1, timeout, true) - defer cancel() - return q.FlushWithContext(ctx) -} - -// Shutdown processing from this queue -func (q *ChannelUniqueQueue) Shutdown() { - log.Trace("ChannelUniqueQueue: %s Shutting down", q.name) - select { - case <-q.shutdownCtx.Done(): - return - default: - } - go func() { - log.Trace("ChannelUniqueQueue: %s Flushing", q.name) - if err := q.FlushWithContext(q.terminateCtx); err != nil { - if !q.IsEmpty() { - log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name) - } - return - } - log.Debug("ChannelUniqueQueue: %s Flushed", q.name) - }() - q.shutdownCtxCancel() - log.Debug("ChannelUniqueQueue: %s Shutdown", q.name) -} - -// Terminate this queue and close the queue -func (q *ChannelUniqueQueue) Terminate() { - log.Trace("ChannelUniqueQueue: %s Terminating", q.name) - q.Shutdown() - select { - case <-q.terminateCtx.Done(): - return - default: - } - q.terminateCtxCancel() - q.baseCtxFinished() - log.Debug("ChannelUniqueQueue: %s Terminated", q.name) -} - -// Name returns the name of this queue -func (q *ChannelUniqueQueue) Name() string { - return q.name -} - -func init() { - queuesMap[ChannelUniqueQueueType] = NewChannelUniqueQueue -} diff --git a/modules/queue/unique_queue_channel_test.go b/modules/queue/unique_queue_channel_test.go deleted file mode 100644 index 824015b834f..00000000000 --- a/modules/queue/unique_queue_channel_test.go +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/log" - - "github.com/stretchr/testify/assert" -) - -func TestChannelUniqueQueue(t *testing.T) { - _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - nilFn := func(_ func()) {} - - queue, err := NewChannelUniqueQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 0, - MaxWorkers: 10, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 5, - Name: "TestChannelQueue", - }, - Workers: 0, - }, &testData{}) - assert.NoError(t, err) - - assert.Equal(t, queue.(*ChannelUniqueQueue).WorkerPool.boostWorkers, 5) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - go queue.Push(&test1) - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) -} - -func TestChannelUniqueQueue_Batch(t *testing.T) { - _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) - - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - nilFn := func(_ func()) {} - - queue, err := NewChannelUniqueQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 2, - BlockTimeout: 0, - BoostTimeout: 0, - BoostWorkers: 0, - MaxWorkers: 10, - }, - Workers: 1, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - queue.Push(&test1) - go queue.Push(&test2) - - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - result2 := <-handleChan - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) -} - -func TestChannelUniqueQueue_Pause(t *testing.T) { - _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) - - lock := sync.Mutex{} - var queue Queue - var err error - pushBack := false - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - lock.Lock() - if pushBack { - if pausable, ok := queue.(Pausable); ok { - pausable.Pause() - } - pushBack = false - lock.Unlock() - return data - } - lock.Unlock() - - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - nilFn := func(_ func()) {} - - queue, err = NewChannelUniqueQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 1, - BlockTimeout: 0, - BoostTimeout: 0, - BoostWorkers: 0, - MaxWorkers: 10, - }, - Workers: 1, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - queue.Push(&test1) - - pausable, ok := queue.(Pausable) - if !assert.True(t, ok) { - return - } - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - pausable.Pause() - - paused, resumed := pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-resumed: - assert.Fail(t, "Queue should not be resumed") - return - default: - assert.Fail(t, "Queue is not paused") - return - } - - queue.Push(&test2) - - var result2 *testData - select { - case result2 = <-handleChan: - assert.Fail(t, "handler chan should be empty") - case <-time.After(100 * time.Millisecond): - } - - assert.Nil(t, result2) - - pausable.Resume() - - select { - case <-resumed: - default: - assert.Fail(t, "Queue should be resumed") - } - - select { - case result2 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test2") - } - - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - lock.Lock() - pushBack = true - lock.Unlock() - - paused, resumed = pausable.IsPausedIsResumed() - - select { - case <-paused: - assert.Fail(t, "Queue should not be paused") - return - case <-resumed: - default: - assert.Fail(t, "Queue is not resumed") - return - } - - queue.Push(&test1) - - select { - case <-paused: - case <-handleChan: - assert.Fail(t, "handler chan should not contain test1") - return - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "queue should be paused") - return - } - - paused, resumed = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-resumed: - assert.Fail(t, "Queue should not be resumed") - return - default: - assert.Fail(t, "Queue is not paused") - return - } - - pausable.Resume() - - select { - case <-resumed: - default: - assert.Fail(t, "Queue should be resumed") - } - - select { - case result1 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test1") - } - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) -} diff --git a/modules/queue/unique_queue_disk.go b/modules/queue/unique_queue_disk.go deleted file mode 100644 index 406f64784ce..00000000000 --- a/modules/queue/unique_queue_disk.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - - "code.gitea.io/gitea/modules/nosql" - - "gitea.com/lunny/levelqueue" -) - -// LevelUniqueQueueType is the type for level queue -const LevelUniqueQueueType Type = "unique-level" - -// LevelUniqueQueueConfiguration is the configuration for a LevelUniqueQueue -type LevelUniqueQueueConfiguration struct { - ByteFIFOQueueConfiguration - DataDir string - ConnectionString string - QueueName string -} - -// LevelUniqueQueue implements a disk library queue -type LevelUniqueQueue struct { - *ByteFIFOUniqueQueue -} - -// NewLevelUniqueQueue creates a ledis local queue -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -func NewLevelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(LevelUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(LevelUniqueQueueConfiguration) - - if len(config.ConnectionString) == 0 { - config.ConnectionString = config.DataDir - } - config.WaitOnEmpty = true - - byteFIFO, err := NewLevelUniqueQueueByteFIFO(config.ConnectionString, config.QueueName) - if err != nil { - return nil, err - } - - byteFIFOQueue, err := NewByteFIFOUniqueQueue(LevelUniqueQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) - if err != nil { - return nil, err - } - - queue := &LevelUniqueQueue{ - ByteFIFOUniqueQueue: byteFIFOQueue, - } - queue.qid = GetManager().Add(queue, LevelUniqueQueueType, config, exemplar) - return queue, nil -} - -var _ UniqueByteFIFO = &LevelUniqueQueueByteFIFO{} - -// LevelUniqueQueueByteFIFO represents a ByteFIFO formed from a LevelUniqueQueue -type LevelUniqueQueueByteFIFO struct { - internal *levelqueue.UniqueQueue - connection string -} - -// NewLevelUniqueQueueByteFIFO creates a new ByteFIFO formed from a LevelUniqueQueue -func NewLevelUniqueQueueByteFIFO(connection, prefix string) (*LevelUniqueQueueByteFIFO, error) { - db, err := nosql.GetManager().GetLevelDB(connection) - if err != nil { - return nil, err - } - - internal, err := levelqueue.NewUniqueQueue(db, []byte(prefix), []byte(prefix+"-unique"), false) - if err != nil { - return nil, err - } - - return &LevelUniqueQueueByteFIFO{ - connection: connection, - internal: internal, - }, nil -} - -// PushFunc pushes data to the end of the fifo and calls the callback if it is added -func (fifo *LevelUniqueQueueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - return fifo.internal.LPushFunc(data, fn) -} - -// PushBack pushes data to the top of the fifo -func (fifo *LevelUniqueQueueByteFIFO) PushBack(ctx context.Context, data []byte) error { - return fifo.internal.RPush(data) -} - -// Pop pops data from the start of the fifo -func (fifo *LevelUniqueQueueByteFIFO) Pop(ctx context.Context) ([]byte, error) { - data, err := fifo.internal.RPop() - if err != nil && err != levelqueue.ErrNotFound { - return nil, err - } - return data, nil -} - -// Len returns the length of the fifo -func (fifo *LevelUniqueQueueByteFIFO) Len(ctx context.Context) int64 { - return fifo.internal.Len() -} - -// Has returns whether the fifo contains this data -func (fifo *LevelUniqueQueueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { - return fifo.internal.Has(data) -} - -// Close this fifo -func (fifo *LevelUniqueQueueByteFIFO) Close() error { - err := fifo.internal.Close() - _ = nosql.GetManager().CloseLevelDB(fifo.connection) - return err -} - -func init() { - queuesMap[LevelUniqueQueueType] = NewLevelUniqueQueue -} diff --git a/modules/queue/unique_queue_disk_channel.go b/modules/queue/unique_queue_disk_channel.go deleted file mode 100644 index cc8a807c672..00000000000 --- a/modules/queue/unique_queue_disk_channel.go +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "runtime/pprof" - "sync" - "time" - - "code.gitea.io/gitea/modules/log" -) - -// PersistableChannelUniqueQueueType is the type for persistable queue -const PersistableChannelUniqueQueueType Type = "unique-persistable-channel" - -// PersistableChannelUniqueQueueConfiguration is the configuration for a PersistableChannelUniqueQueue -type PersistableChannelUniqueQueueConfiguration struct { - Name string - DataDir string - BatchLength int - QueueLength int - Timeout time.Duration - MaxAttempts int - Workers int - MaxWorkers int - BlockTimeout time.Duration - BoostTimeout time.Duration - BoostWorkers int -} - -// PersistableChannelUniqueQueue wraps a channel queue and level queue together -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -type PersistableChannelUniqueQueue struct { - channelQueue *ChannelUniqueQueue - delayedStarter - lock sync.Mutex - closed chan struct{} -} - -// NewPersistableChannelUniqueQueue creates a wrapped batched channel queue with persistable level queue backend when shutting down -// This differs from a wrapped queue in that the persistent queue is only used to persist at shutdown/terminate -func NewPersistableChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(PersistableChannelUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(PersistableChannelUniqueQueueConfiguration) - - queue := &PersistableChannelUniqueQueue{ - closed: make(chan struct{}), - } - - wrappedHandle := func(data ...Data) (failed []Data) { - for _, unhandled := range handle(data...) { - if fail := queue.PushBack(unhandled); fail != nil { - failed = append(failed, fail) - } - } - return failed - } - - channelUniqueQueue, err := NewChannelUniqueQueue(wrappedHandle, ChannelUniqueQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: config.QueueLength, - BatchLength: config.BatchLength, - BlockTimeout: config.BlockTimeout, - BoostTimeout: config.BoostTimeout, - BoostWorkers: config.BoostWorkers, - MaxWorkers: config.MaxWorkers, - Name: config.Name + "-channel", - }, - Workers: config.Workers, - }, exemplar) - if err != nil { - return nil, err - } - - // the level backend only needs temporary workers to catch up with the previously dropped work - levelCfg := LevelUniqueQueueConfiguration{ - ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: config.QueueLength, - BatchLength: config.BatchLength, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 1, - MaxWorkers: 5, - Name: config.Name + "-level", - }, - Workers: 0, - }, - DataDir: config.DataDir, - QueueName: config.Name + "-level", - } - - queue.channelQueue = channelUniqueQueue.(*ChannelUniqueQueue) - - levelQueue, err := NewLevelUniqueQueue(func(data ...Data) []Data { - for _, datum := range data { - err := queue.Push(datum) - if err != nil && err != ErrAlreadyInQueue { - log.Error("Unable push to channelled queue: %v", err) - } - } - return nil - }, levelCfg, exemplar) - if err == nil { - queue.delayedStarter = delayedStarter{ - internal: levelQueue.(*LevelUniqueQueue), - name: config.Name, - } - - _ = GetManager().Add(queue, PersistableChannelUniqueQueueType, config, exemplar) - return queue, nil - } - if IsErrInvalidConfiguration(err) { - // Retrying ain't gonna make this any better... - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - queue.delayedStarter = delayedStarter{ - cfg: levelCfg, - underlying: LevelUniqueQueueType, - timeout: config.Timeout, - maxAttempts: config.MaxAttempts, - name: config.Name, - } - _ = GetManager().Add(queue, PersistableChannelUniqueQueueType, config, exemplar) - return queue, nil -} - -// Name returns the name of this queue -func (q *PersistableChannelUniqueQueue) Name() string { - return q.delayedStarter.name -} - -// Push will push the indexer data to queue -func (q *PersistableChannelUniqueQueue) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushFunc will push the indexer data to queue -func (q *PersistableChannelUniqueQueue) PushFunc(data Data, fn func() error) error { - select { - case <-q.closed: - return q.internal.(UniqueQueue).PushFunc(data, fn) - default: - return q.channelQueue.PushFunc(data, fn) - } -} - -// PushBack will push the indexer data to queue -func (q *PersistableChannelUniqueQueue) PushBack(data Data) error { - select { - case <-q.closed: - if pbr, ok := q.internal.(PushBackable); ok { - return pbr.PushBack(data) - } - return q.internal.Push(data) - default: - return q.channelQueue.Push(data) - } -} - -// Has will test if the queue has the data -func (q *PersistableChannelUniqueQueue) Has(data Data) (bool, error) { - // This is more difficult... - has, err := q.channelQueue.Has(data) - if err != nil || has { - return has, err - } - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return false, nil - } - return q.internal.(UniqueQueue).Has(data) -} - -// Run starts to run the queue -func (q *PersistableChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.channelQueue.baseCtx) - log.Debug("PersistableChannelUniqueQueue: %s Starting", q.delayedStarter.name) - - q.lock.Lock() - if q.internal == nil { - err := q.setInternal(atShutdown, func(data ...Data) []Data { - for _, datum := range data { - err := q.Push(datum) - if err != nil && err != ErrAlreadyInQueue { - log.Error("Unable push to channelled queue: %v", err) - } - } - return nil - }, q.channelQueue.exemplar) - q.lock.Unlock() - if err != nil { - log.Fatal("Unable to create internal queue for %s Error: %v", q.Name(), err) - return - } - } else { - q.lock.Unlock() - } - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - _ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0) - - if luq, ok := q.internal.(*LevelUniqueQueue); ok && !luq.IsEmpty() { - // Just run the level queue - we shut it down once it's flushed - go luq.Run(func(_ func()) {}, func(_ func()) {}) - go func() { - _ = luq.Flush(0) - for !luq.IsEmpty() { - _ = luq.Flush(0) - select { - case <-time.After(100 * time.Millisecond): - case <-luq.shutdownCtx.Done(): - if luq.byteFIFO.Len(luq.terminateCtx) > 0 { - log.Warn("LevelUniqueQueue: %s shut down before completely flushed", luq.Name()) - } - return - } - } - log.Debug("LevelUniqueQueue: %s flushed so shutting down", luq.Name()) - luq.Shutdown() - GetManager().Remove(luq.qid) - }() - } else { - log.Debug("PersistableChannelUniqueQueue: %s Skipping running the empty level queue", q.delayedStarter.name) - _ = q.internal.Flush(0) - q.internal.(*LevelUniqueQueue).Shutdown() - GetManager().Remove(q.internal.(*LevelUniqueQueue).qid) - } -} - -// Flush flushes the queue -func (q *PersistableChannelUniqueQueue) Flush(timeout time.Duration) error { - return q.channelQueue.Flush(timeout) -} - -// FlushWithContext flushes the queue -func (q *PersistableChannelUniqueQueue) FlushWithContext(ctx context.Context) error { - return q.channelQueue.FlushWithContext(ctx) -} - -// IsEmpty checks if a queue is empty -func (q *PersistableChannelUniqueQueue) IsEmpty() bool { - return q.channelQueue.IsEmpty() -} - -// IsPaused will return if the pool or queue is paused -func (q *PersistableChannelUniqueQueue) IsPaused() bool { - return q.channelQueue.IsPaused() -} - -// Pause will pause the pool or queue -func (q *PersistableChannelUniqueQueue) Pause() { - q.channelQueue.Pause() -} - -// Resume will resume the pool or queue -func (q *PersistableChannelUniqueQueue) Resume() { - q.channelQueue.Resume() -} - -// IsPausedIsResumed will return a bool indicating if the pool or queue is paused and a channel that will be closed when it is resumed -func (q *PersistableChannelUniqueQueue) IsPausedIsResumed() (paused, resumed <-chan struct{}) { - return q.channelQueue.IsPausedIsResumed() -} - -// Shutdown processing this queue -func (q *PersistableChannelUniqueQueue) Shutdown() { - log.Trace("PersistableChannelUniqueQueue: %s Shutting down", q.delayedStarter.name) - q.lock.Lock() - select { - case <-q.closed: - q.lock.Unlock() - return - default: - if q.internal != nil { - q.internal.(*LevelUniqueQueue).Shutdown() - } - close(q.closed) - q.lock.Unlock() - } - - log.Trace("PersistableChannelUniqueQueue: %s Cancelling pools", q.delayedStarter.name) - q.internal.(*LevelUniqueQueue).baseCtxCancel() - q.channelQueue.baseCtxCancel() - log.Trace("PersistableChannelUniqueQueue: %s Waiting til done", q.delayedStarter.name) - q.channelQueue.Wait() - q.internal.(*LevelUniqueQueue).Wait() - // Redirect all remaining data in the chan to the internal channel - close(q.channelQueue.dataChan) - log.Trace("PersistableChannelUniqueQueue: %s Redirecting remaining data", q.delayedStarter.name) - countOK, countLost := 0, 0 - for data := range q.channelQueue.dataChan { - err := q.internal.(*LevelUniqueQueue).Push(data) - if err != nil { - log.Error("PersistableChannelUniqueQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err) - countLost++ - } else { - countOK++ - } - } - if countLost > 0 { - log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost) - } else if countOK > 0 { - log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK) - } - log.Trace("PersistableChannelUniqueQueue: %s Done Redirecting remaining data", q.delayedStarter.name) - - log.Debug("PersistableChannelUniqueQueue: %s Shutdown", q.delayedStarter.name) -} - -// Terminate this queue and close the queue -func (q *PersistableChannelUniqueQueue) Terminate() { - log.Trace("PersistableChannelUniqueQueue: %s Terminating", q.delayedStarter.name) - q.Shutdown() - q.lock.Lock() - defer q.lock.Unlock() - if q.internal != nil { - q.internal.(*LevelUniqueQueue).Terminate() - } - q.channelQueue.baseCtxFinished() - log.Debug("PersistableChannelUniqueQueue: %s Terminated", q.delayedStarter.name) -} - -func init() { - queuesMap[PersistableChannelUniqueQueueType] = NewPersistableChannelUniqueQueue -} diff --git a/modules/queue/unique_queue_disk_channel_test.go b/modules/queue/unique_queue_disk_channel_test.go deleted file mode 100644 index e2fe4aceee5..00000000000 --- a/modules/queue/unique_queue_disk_channel_test.go +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "os" - "strconv" - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/log" - - "github.com/stretchr/testify/assert" -) - -func TestPersistableChannelUniqueQueue(t *testing.T) { - if os.Getenv("CI") != "" { - t.Skip("Skipping because test is flaky on CI") - } - - tmpDir := t.TempDir() - _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) - - // Common function to create the Queue - newQueue := func(name string, handle func(data ...Data) []Data) Queue { - q, err := NewPersistableChannelUniqueQueue(handle, - PersistableChannelUniqueQueueConfiguration{ - Name: name, - DataDir: tmpDir, - QueueLength: 200, - MaxWorkers: 1, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 1, - Workers: 0, - }, "task-0") - assert.NoError(t, err) - return q - } - - // runs the provided queue and provides some timer function - type channels struct { - readyForShutdown chan struct{} // closed when shutdown functions have been assigned - readyForTerminate chan struct{} // closed when terminate functions have been assigned - signalShutdown chan struct{} // Should close to signal shutdown - doneShutdown chan struct{} // closed when shutdown function is done - queueTerminate []func() // list of atTerminate functions to call atTerminate - need to be accessed with lock - } - runQueue := func(q Queue, lock *sync.Mutex) *channels { - chans := &channels{ - readyForShutdown: make(chan struct{}), - readyForTerminate: make(chan struct{}), - signalShutdown: make(chan struct{}), - doneShutdown: make(chan struct{}), - } - go q.Run(func(atShutdown func()) { - go func() { - lock.Lock() - select { - case <-chans.readyForShutdown: - default: - close(chans.readyForShutdown) - } - lock.Unlock() - <-chans.signalShutdown - atShutdown() - close(chans.doneShutdown) - }() - }, func(atTerminate func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-chans.readyForTerminate: - default: - close(chans.readyForTerminate) - } - chans.queueTerminate = append(chans.queueTerminate, atTerminate) - }) - - return chans - } - - // call to shutdown and terminate the queue associated with the channels - doTerminate := func(chans *channels, lock *sync.Mutex) { - <-chans.readyForTerminate - - lock.Lock() - callbacks := []func(){} - callbacks = append(callbacks, chans.queueTerminate...) - lock.Unlock() - - for _, callback := range callbacks { - callback() - } - } - - mapLock := sync.Mutex{} - executedInitial := map[string][]string{} - hasInitial := map[string][]string{} - - fillQueue := func(name string, done chan struct{}) { - t.Run("Initial Filling: "+name, func(t *testing.T) { - lock := sync.Mutex{} - - startAt100Queued := make(chan struct{}) - stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item - - handle := func(data ...Data) []Data { - <-startAt100Queued - for _, datum := range data { - s := datum.(string) - mapLock.Lock() - executedInitial[name] = append(executedInitial[name], s) - mapLock.Unlock() - if s == "task-20" { - close(stopAt20Shutdown) - } - } - return nil - } - - q := newQueue(name, handle) - - // add 100 tasks to the queue - for i := 0; i < 100; i++ { - _ = q.Push("task-" + strconv.Itoa(i)) - } - close(startAt100Queued) - - chans := runQueue(q, &lock) - - <-chans.readyForShutdown - <-stopAt20Shutdown - close(chans.signalShutdown) - <-chans.doneShutdown - _ = q.Push("final") - - // check which tasks are still in the queue - for i := 0; i < 100; i++ { - if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has { - mapLock.Lock() - hasInitial[name] = append(hasInitial[name], "task-"+strconv.Itoa(i)) - mapLock.Unlock() - } - } - if has, _ := q.(UniqueQueue).Has("final"); has { - mapLock.Lock() - hasInitial[name] = append(hasInitial[name], "final") - mapLock.Unlock() - } else { - assert.Fail(t, "UnqueQueue %s should have \"final\"", name) - } - doTerminate(chans, &lock) - mapLock.Lock() - assert.Equal(t, 101, len(executedInitial[name])+len(hasInitial[name])) - mapLock.Unlock() - }) - close(done) - } - - doneA := make(chan struct{}) - doneB := make(chan struct{}) - - go fillQueue("QueueA", doneA) - go fillQueue("QueueB", doneB) - - <-doneA - <-doneB - - executedEmpty := map[string][]string{} - hasEmpty := map[string][]string{} - emptyQueue := func(name string, done chan struct{}) { - t.Run("Empty Queue: "+name, func(t *testing.T) { - lock := sync.Mutex{} - stop := make(chan struct{}) - - // collect the tasks that have been executed - handle := func(data ...Data) []Data { - lock.Lock() - for _, datum := range data { - mapLock.Lock() - executedEmpty[name] = append(executedEmpty[name], datum.(string)) - mapLock.Unlock() - if datum.(string) == "final" { - close(stop) - } - } - lock.Unlock() - return nil - } - - q := newQueue(name, handle) - chans := runQueue(q, &lock) - - <-chans.readyForShutdown - <-stop - close(chans.signalShutdown) - <-chans.doneShutdown - - // check which tasks are still in the queue - for i := 0; i < 100; i++ { - if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has { - mapLock.Lock() - hasEmpty[name] = append(hasEmpty[name], "task-"+strconv.Itoa(i)) - mapLock.Unlock() - } - } - doTerminate(chans, &lock) - - mapLock.Lock() - assert.Equal(t, 101, len(executedInitial[name])+len(executedEmpty[name])) - assert.Empty(t, hasEmpty[name]) - mapLock.Unlock() - }) - close(done) - } - - doneA = make(chan struct{}) - doneB = make(chan struct{}) - - go emptyQueue("QueueA", doneA) - go emptyQueue("QueueB", doneB) - - <-doneA - <-doneB - - mapLock.Lock() - t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v", - len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"])) - - // reset and rerun - executedInitial = map[string][]string{} - hasInitial = map[string][]string{} - executedEmpty = map[string][]string{} - hasEmpty = map[string][]string{} - mapLock.Unlock() - - doneA = make(chan struct{}) - doneB = make(chan struct{}) - - go fillQueue("QueueA", doneA) - go fillQueue("QueueB", doneB) - - <-doneA - <-doneB - - doneA = make(chan struct{}) - doneB = make(chan struct{}) - - go emptyQueue("QueueA", doneA) - go emptyQueue("QueueB", doneB) - - <-doneA - <-doneB - - mapLock.Lock() - t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v", - len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"])) - mapLock.Unlock() -} diff --git a/modules/queue/unique_queue_redis.go b/modules/queue/unique_queue_redis.go deleted file mode 100644 index ae1df08ebd5..00000000000 --- a/modules/queue/unique_queue_redis.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - - "github.com/redis/go-redis/v9" -) - -// RedisUniqueQueueType is the type for redis queue -const RedisUniqueQueueType Type = "unique-redis" - -// RedisUniqueQueue redis queue -type RedisUniqueQueue struct { - *ByteFIFOUniqueQueue -} - -// RedisUniqueQueueConfiguration is the configuration for the redis queue -type RedisUniqueQueueConfiguration struct { - ByteFIFOQueueConfiguration - RedisUniqueByteFIFOConfiguration -} - -// NewRedisUniqueQueue creates single redis or cluster redis queue. -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -func NewRedisUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(RedisUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(RedisUniqueQueueConfiguration) - - byteFIFO, err := NewRedisUniqueByteFIFO(config.RedisUniqueByteFIFOConfiguration) - if err != nil { - return nil, err - } - - if len(byteFIFO.setName) == 0 { - byteFIFO.setName = byteFIFO.queueName + "_unique" - } - - byteFIFOQueue, err := NewByteFIFOUniqueQueue(RedisUniqueQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) - if err != nil { - return nil, err - } - - queue := &RedisUniqueQueue{ - ByteFIFOUniqueQueue: byteFIFOQueue, - } - - queue.qid = GetManager().Add(queue, RedisUniqueQueueType, config, exemplar) - - return queue, nil -} - -var _ UniqueByteFIFO = &RedisUniqueByteFIFO{} - -// RedisUniqueByteFIFO represents a UniqueByteFIFO formed from a redisClient -type RedisUniqueByteFIFO struct { - RedisByteFIFO - setName string -} - -// RedisUniqueByteFIFOConfiguration is the configuration for the RedisUniqueByteFIFO -type RedisUniqueByteFIFOConfiguration struct { - RedisByteFIFOConfiguration - SetName string -} - -// NewRedisUniqueByteFIFO creates a UniqueByteFIFO formed from a redisClient -func NewRedisUniqueByteFIFO(config RedisUniqueByteFIFOConfiguration) (*RedisUniqueByteFIFO, error) { - internal, err := NewRedisByteFIFO(config.RedisByteFIFOConfiguration) - if err != nil { - return nil, err - } - - fifo := &RedisUniqueByteFIFO{ - RedisByteFIFO: *internal, - setName: config.SetName, - } - - return fifo, nil -} - -// PushFunc pushes data to the end of the fifo and calls the callback if it is added -func (fifo *RedisUniqueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - added, err := fifo.client.SAdd(ctx, fifo.setName, data).Result() - if err != nil { - return err - } - if added == 0 { - return ErrAlreadyInQueue - } - if fn != nil { - if err := fn(); err != nil { - return err - } - } - return fifo.client.RPush(ctx, fifo.queueName, data).Err() -} - -// PushBack pushes data to the top of the fifo -func (fifo *RedisUniqueByteFIFO) PushBack(ctx context.Context, data []byte) error { - added, err := fifo.client.SAdd(ctx, fifo.setName, data).Result() - if err != nil { - return err - } - if added == 0 { - return ErrAlreadyInQueue - } - return fifo.client.LPush(ctx, fifo.queueName, data).Err() -} - -// Pop pops data from the start of the fifo -func (fifo *RedisUniqueByteFIFO) Pop(ctx context.Context) ([]byte, error) { - data, err := fifo.client.LPop(ctx, fifo.queueName).Bytes() - if err != nil && err != redis.Nil { - return data, err - } - - if len(data) == 0 { - return data, nil - } - - err = fifo.client.SRem(ctx, fifo.setName, data).Err() - return data, err -} - -// Has returns whether the fifo contains this data -func (fifo *RedisUniqueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { - return fifo.client.SIsMember(ctx, fifo.setName, data).Result() -} - -func init() { - queuesMap[RedisUniqueQueueType] = NewRedisUniqueQueue -} diff --git a/modules/queue/unique_queue_wrapped.go b/modules/queue/unique_queue_wrapped.go deleted file mode 100644 index 22eeb75c401..00000000000 --- a/modules/queue/unique_queue_wrapped.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "fmt" - "sync" - "time" -) - -// WrappedUniqueQueueType is the type for a wrapped delayed starting queue -const WrappedUniqueQueueType Type = "unique-wrapped" - -// WrappedUniqueQueueConfiguration is the configuration for a WrappedUniqueQueue -type WrappedUniqueQueueConfiguration struct { - Underlying Type - Timeout time.Duration - MaxAttempts int - Config interface{} - QueueLength int - Name string -} - -// WrappedUniqueQueue wraps a delayed starting unique queue -type WrappedUniqueQueue struct { - *WrappedQueue - table map[Data]bool - tlock sync.Mutex - ready bool -} - -// NewWrappedUniqueQueue will attempt to create a unique queue of the provided type, -// but if there is a problem creating this queue it will instead create -// a WrappedUniqueQueue with delayed startup of the queue instead and a -// channel which will be redirected to the queue -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -func NewWrappedUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(WrappedUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(WrappedUniqueQueueConfiguration) - - queue, err := NewQueue(config.Underlying, handle, config.Config, exemplar) - if err == nil { - // Just return the queue there is no need to wrap - return queue, nil - } - if IsErrInvalidConfiguration(err) { - // Retrying ain't gonna make this any better... - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - wrapped := &WrappedUniqueQueue{ - WrappedQueue: &WrappedQueue{ - channel: make(chan Data, config.QueueLength), - exemplar: exemplar, - delayedStarter: delayedStarter{ - cfg: config.Config, - underlying: config.Underlying, - timeout: config.Timeout, - maxAttempts: config.MaxAttempts, - name: config.Name, - }, - }, - table: map[Data]bool{}, - } - - // wrapped.handle is passed to the delayedStarting internal queue and is run to handle - // data passed to - wrapped.handle = func(data ...Data) (unhandled []Data) { - for _, datum := range data { - wrapped.tlock.Lock() - if !wrapped.ready { - delete(wrapped.table, data) - // If our table is empty all of the requests we have buffered between the - // wrapper queue starting and the internal queue starting have been handled. - // We can stop buffering requests in our local table and just pass Push - // direct to the internal queue - if len(wrapped.table) == 0 { - wrapped.ready = true - } - } - wrapped.tlock.Unlock() - if u := handle(datum); u != nil { - unhandled = append(unhandled, u...) - } - } - return unhandled - } - _ = GetManager().Add(queue, WrappedUniqueQueueType, config, exemplar) - return wrapped, nil -} - -// Push will push the data to the internal channel checking it against the exemplar -func (q *WrappedUniqueQueue) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushFunc will push the data to the internal channel checking it against the exemplar -func (q *WrappedUniqueQueue) PushFunc(data Data, fn func() error) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - - q.tlock.Lock() - if q.ready { - // ready means our table is empty and all of the requests we have buffered between the - // wrapper queue starting and the internal queue starting have been handled. - // We can stop buffering requests in our local table and just pass Push - // direct to the internal queue - q.tlock.Unlock() - return q.internal.(UniqueQueue).PushFunc(data, fn) - } - - locked := true - defer func() { - if locked { - q.tlock.Unlock() - } - }() - if _, ok := q.table[data]; ok { - return ErrAlreadyInQueue - } - // FIXME: We probably need to implement some sort of limit here - // If the downstream queue blocks this table will grow without limit - q.table[data] = true - if fn != nil { - err := fn() - if err != nil { - delete(q.table, data) - return err - } - } - locked = false - q.tlock.Unlock() - - q.channel <- data - return nil -} - -// Has checks if the data is in the queue -func (q *WrappedUniqueQueue) Has(data Data) (bool, error) { - q.tlock.Lock() - defer q.tlock.Unlock() - if q.ready { - return q.internal.(UniqueQueue).Has(data) - } - _, has := q.table[data] - return has, nil -} - -// IsEmpty checks whether the queue is empty -func (q *WrappedUniqueQueue) IsEmpty() bool { - q.tlock.Lock() - if len(q.table) > 0 { - q.tlock.Unlock() - return false - } - if q.ready { - q.tlock.Unlock() - return q.internal.IsEmpty() - } - q.tlock.Unlock() - return false -} - -func init() { - queuesMap[WrappedUniqueQueueType] = NewWrappedUniqueQueue -} diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go new file mode 100644 index 00000000000..7127ea1117d --- /dev/null +++ b/modules/queue/workergroup.go @@ -0,0 +1,331 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "code.gitea.io/gitea/modules/log" +) + +var ( + infiniteTimerC = make(chan time.Time) + batchDebounceDuration = 100 * time.Millisecond + workerIdleDuration = 1 * time.Second + + unhandledItemRequeueDuration atomic.Int64 // to avoid data race during test +) + +func init() { + unhandledItemRequeueDuration.Store(int64(5 * time.Second)) +} + +// workerGroup is a group of workers to work with a WorkerPoolQueue +type workerGroup[T any] struct { + q *WorkerPoolQueue[T] + wg sync.WaitGroup + + ctxWorker context.Context + ctxWorkerCancel context.CancelFunc + + batchBuffer []T + popItemChan chan []byte + popItemErr chan error +} + +func (wg *workerGroup[T]) doPrepareWorkerContext() { + wg.ctxWorker, wg.ctxWorkerCancel = context.WithCancel(wg.q.ctxRun) +} + +// doDispatchBatchToWorker dispatches a batch of items to worker's channel. +// If the channel is full, it tries to start a new worker if possible. +func (q *WorkerPoolQueue[T]) doDispatchBatchToWorker(wg *workerGroup[T], flushChan chan flushType) { + batch := wg.batchBuffer + wg.batchBuffer = nil + + if len(batch) == 0 { + return + } + + full := false + select { + case q.batchChan <- batch: + default: + full = true + } + + q.workerNumMu.Lock() + noWorker := q.workerNum == 0 + if full || noWorker { + if q.workerNum < q.workerMaxNum || noWorker && q.workerMaxNum <= 0 { + q.workerNum++ + q.doStartNewWorker(wg) + } + } + q.workerNumMu.Unlock() + + if full { + select { + case q.batchChan <- batch: + case flush := <-flushChan: + q.doWorkerHandle(batch) + q.doFlush(wg, flush) + case <-q.ctxRun.Done(): + wg.batchBuffer = batch // return the batch to buffer, the "doRun" function will handle it + } + } +} + +// doWorkerHandle calls the safeHandler to handle a batch of items, and it increases/decreases the active worker number. +// If the context has been canceled, it should not be caller because the "Push" still needs the context, in such case, call q.safeHandler directly +func (q *WorkerPoolQueue[T]) doWorkerHandle(batch []T) { + q.workerNumMu.Lock() + q.workerActiveNum++ + q.workerNumMu.Unlock() + + defer func() { + q.workerNumMu.Lock() + q.workerActiveNum-- + q.workerNumMu.Unlock() + }() + + unhandled := q.safeHandler(batch...) + // if none of the items were handled, it should back-off for a few seconds + // in this case the handler (eg: document indexer) may have encountered some errors/failures + if len(unhandled) == len(batch) && unhandledItemRequeueDuration.Load() != 0 { + log.Error("Queue %q failed to handle batch of %d items, backoff for a few seconds", q.GetName(), len(batch)) + select { + case <-q.ctxRun.Done(): + case <-time.After(time.Duration(unhandledItemRequeueDuration.Load())): + } + } + for _, item := range unhandled { + if err := q.Push(item); err != nil { + if !q.basePushForShutdown(item) { + log.Error("Failed to requeue item for queue %q when calling handler: %v", q.GetName(), err) + } + } + } +} + +// basePushForShutdown tries to requeue items into the base queue when the WorkerPoolQueue is shutting down. +// If the queue is shutting down, it returns true and try to push the items +// Otherwise it does nothing and returns false +func (q *WorkerPoolQueue[T]) basePushForShutdown(items ...T) bool { + ctxShutdown := q.ctxShutdown.Load() + if ctxShutdown == nil { + return false + } + for _, item := range items { + // if there is still any error, the queue can do nothing instead of losing the items + if err := q.baseQueue.PushItem(*ctxShutdown, q.marshal(item)); err != nil { + log.Error("Failed to requeue item for queue %q when shutting down: %v", q.GetName(), err) + } + } + return true +} + +// doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. +func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { + wp.wg.Add(1) + + go func() { + defer wp.wg.Done() + + log.Debug("Queue %q starts new worker", q.GetName()) + defer log.Debug("Queue %q stops idle worker", q.GetName()) + + t := time.NewTicker(workerIdleDuration) + keepWorking := true + stopWorking := func() { + q.workerNumMu.Lock() + keepWorking = false + q.workerNum-- + q.workerNumMu.Unlock() + } + for keepWorking { + select { + case <-wp.ctxWorker.Done(): + stopWorking() + case batch, ok := <-q.batchChan: + if !ok { + stopWorking() + } else { + q.doWorkerHandle(batch) + t.Reset(workerIdleDuration) + } + case <-t.C: + q.workerNumMu.Lock() + keepWorking = q.workerNum <= 1 + if !keepWorking { + q.workerNum-- + } + q.workerNumMu.Unlock() + } + } + }() +} + +// doFlush flushes the queue: it tries to read all items from the queue and handles them. +// It is for testing purpose only. It's not designed to work for a cluster. +func (q *WorkerPoolQueue[T]) doFlush(wg *workerGroup[T], flush flushType) { + log.Debug("Queue %q starts flushing", q.GetName()) + defer log.Debug("Queue %q finishes flushing", q.GetName()) + + // stop all workers, and prepare a new worker context to start new workers + + wg.ctxWorkerCancel() + wg.wg.Wait() + + defer func() { + close(flush) + wg.doPrepareWorkerContext() + }() + + // drain the batch channel first +loop: + for { + select { + case batch := <-q.batchChan: + q.doWorkerHandle(batch) + default: + break loop + } + } + + // drain the popItem channel + emptyCounter := 0 + for { + select { + case data, dataOk := <-wg.popItemChan: + if !dataOk { + return + } + emptyCounter = 0 + if v, jsonOk := q.unmarshal(data); !jsonOk { + continue + } else { + q.doWorkerHandle([]T{v}) + } + case err := <-wg.popItemErr: + if !q.isCtxRunCanceled() { + log.Error("Failed to pop item from queue %q (doFlush): %v", q.GetName(), err) + } + return + case <-q.ctxRun.Done(): + log.Debug("Queue %q is shutting down", q.GetName()) + return + case <-time.After(20 * time.Millisecond): + // There is no reliable way to make sure all queue items are consumed by the Flush, there always might be some items stored in some buffers/temp variables. + // If we run Gitea in a cluster, we can even not guarantee all items are consumed in a deterministic instance. + // Luckily, the "Flush" trick is only used in tests, so far so good. + if cnt, _ := q.baseQueue.Len(q.ctxRun); cnt == 0 && len(wg.popItemChan) == 0 { + emptyCounter++ + } + if emptyCounter >= 2 { + return + } + } + } +} + +func (q *WorkerPoolQueue[T]) isCtxRunCanceled() bool { + select { + case <-q.ctxRun.Done(): + return true + default: + return false + } +} + +var skipFlushChan = make(chan flushType) // an empty flush chan, used to skip reading other flush requests + +// doRun is the main loop of the queue. All related "doXxx" functions are executed in its context. +func (q *WorkerPoolQueue[T]) doRun() { + log.Debug("Queue %q starts running", q.GetName()) + defer log.Debug("Queue %q stops running", q.GetName()) + + wg := &workerGroup[T]{q: q} + wg.doPrepareWorkerContext() + wg.popItemChan, wg.popItemErr = popItemByChan(q.ctxRun, q.baseQueue.PopItem) + + defer func() { + q.ctxRunCancel() + + // drain all data on the fly + // since the queue is shutting down, the items can't be dispatched to workers because the context is canceled + // it can't call doWorkerHandle either, because there is no chance to push unhandled items back to the queue + var unhandled []T + close(q.batchChan) + for batch := range q.batchChan { + unhandled = append(unhandled, batch...) + } + unhandled = append(unhandled, wg.batchBuffer...) + for data := range wg.popItemChan { + if v, ok := q.unmarshal(data); ok { + unhandled = append(unhandled, v) + } + } + + ctxShutdownPtr := q.ctxShutdown.Load() + if ctxShutdownPtr != nil { + // if there is a shutdown context, try to push the items back to the base queue + q.basePushForShutdown(unhandled...) + workerDone := make(chan struct{}) + // the only way to wait for the workers, because the handlers do not have context to wait for + go func() { wg.wg.Wait(); close(workerDone) }() + select { + case <-workerDone: + case <-(*ctxShutdownPtr).Done(): + log.Error("Queue %q is shutting down, but workers are still running after timeout", q.GetName()) + } + } else { + // if there is no shutdown context, just call the handler to try to handle the items. if the handler fails again, the items are lost + q.safeHandler(unhandled...) + } + + close(q.shutdownDone) + }() + + var batchDispatchC <-chan time.Time = infiniteTimerC + for { + select { + case data, dataOk := <-wg.popItemChan: + if !dataOk { + return + } + if v, jsonOk := q.unmarshal(data); !jsonOk { + testRecorder.Record("pop:corrupted:%s", data) // in rare cases the levelqueue(leveldb) might be corrupted + continue + } else { + wg.batchBuffer = append(wg.batchBuffer, v) + } + if len(wg.batchBuffer) >= q.batchLength { + q.doDispatchBatchToWorker(wg, q.flushChan) + } else if batchDispatchC == infiniteTimerC { + batchDispatchC = time.After(batchDebounceDuration) + } // else: batchDispatchC is already a debounce timer, it will be triggered soon + case <-batchDispatchC: + batchDispatchC = infiniteTimerC + q.doDispatchBatchToWorker(wg, q.flushChan) + case flush := <-q.flushChan: + // before flushing, it needs to try to dispatch the batch to worker first, in case there is no worker running + // after the flushing, there is at least one worker running, so "doFlush" could wait for workers to finish + // since we are already in a "flush" operation, so the dispatching function shouldn't read the flush chan. + q.doDispatchBatchToWorker(wg, skipFlushChan) + q.doFlush(wg, flush) + case err := <-wg.popItemErr: + if !q.isCtxRunCanceled() { + log.Error("Failed to pop item from queue %q (doRun): %v", q.GetName(), err) + } + return + case <-q.ctxRun.Done(): + log.Debug("Queue %q is shutting down", q.GetName()) + return + } + } +} diff --git a/modules/queue/workerpool.go b/modules/queue/workerpool.go deleted file mode 100644 index b32128cb821..00000000000 --- a/modules/queue/workerpool.go +++ /dev/null @@ -1,613 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/util" -) - -// WorkerPool represent a dynamically growable worker pool for a -// provided handler function. They have an internal channel which -// they use to detect if there is a block and will grow and shrink in -// response to demand as per configuration. -type WorkerPool struct { - // This field requires to be the first one in the struct. - // This is to allow 64 bit atomic operations on 32-bit machines. - // See: https://pkg.go.dev/sync/atomic#pkg-note-BUG & Gitea issue 19518 - numInQueue int64 - lock sync.Mutex - baseCtx context.Context - baseCtxCancel context.CancelFunc - baseCtxFinished process.FinishedFunc - paused chan struct{} - resumed chan struct{} - cond *sync.Cond - qid int64 - maxNumberOfWorkers int - numberOfWorkers int - batchLength int - handle HandlerFunc - dataChan chan Data - blockTimeout time.Duration - boostTimeout time.Duration - boostWorkers int -} - -var ( - _ Flushable = &WorkerPool{} - _ ManagedPool = &WorkerPool{} -) - -// WorkerPoolConfiguration is the basic configuration for a WorkerPool -type WorkerPoolConfiguration struct { - Name string - QueueLength int - BatchLength int - BlockTimeout time.Duration - BoostTimeout time.Duration - BoostWorkers int - MaxWorkers int -} - -// NewWorkerPool creates a new worker pool -func NewWorkerPool(handle HandlerFunc, config WorkerPoolConfiguration) *WorkerPool { - ctx, cancel, finished := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Queue: %s", config.Name), process.SystemProcessType, false) - - dataChan := make(chan Data, config.QueueLength) - pool := &WorkerPool{ - baseCtx: ctx, - baseCtxCancel: cancel, - baseCtxFinished: finished, - batchLength: config.BatchLength, - dataChan: dataChan, - resumed: closedChan, - paused: make(chan struct{}), - handle: handle, - blockTimeout: config.BlockTimeout, - boostTimeout: config.BoostTimeout, - boostWorkers: config.BoostWorkers, - maxNumberOfWorkers: config.MaxWorkers, - } - - return pool -} - -// Done returns when this worker pool's base context has been cancelled -func (p *WorkerPool) Done() <-chan struct{} { - return p.baseCtx.Done() -} - -// Push pushes the data to the internal channel -func (p *WorkerPool) Push(data Data) { - atomic.AddInt64(&p.numInQueue, 1) - p.lock.Lock() - select { - case <-p.paused: - p.lock.Unlock() - p.dataChan <- data - return - default: - } - - if p.blockTimeout > 0 && p.boostTimeout > 0 && (p.numberOfWorkers <= p.maxNumberOfWorkers || p.maxNumberOfWorkers < 0) { - if p.numberOfWorkers == 0 { - p.zeroBoost() - } else { - p.lock.Unlock() - } - p.pushBoost(data) - } else { - p.lock.Unlock() - p.dataChan <- data - } -} - -// HasNoWorkerScaling will return true if the queue has no workers, and has no worker boosting -func (p *WorkerPool) HasNoWorkerScaling() bool { - p.lock.Lock() - defer p.lock.Unlock() - return p.hasNoWorkerScaling() -} - -func (p *WorkerPool) hasNoWorkerScaling() bool { - return p.numberOfWorkers == 0 && (p.boostTimeout == 0 || p.boostWorkers == 0 || p.maxNumberOfWorkers == 0) -} - -// zeroBoost will add a temporary boost worker for a no worker queue -// p.lock must be locked at the start of this function BUT it will be unlocked by the end of this function -// (This is because addWorkers has to be called whilst unlocked) -func (p *WorkerPool) zeroBoost() { - ctx, cancel := context.WithTimeout(p.baseCtx, p.boostTimeout) - mq := GetManager().GetManagedQueue(p.qid) - boost := p.boostWorkers - if (boost+p.numberOfWorkers) > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0 { - boost = p.maxNumberOfWorkers - p.numberOfWorkers - } - if mq != nil { - log.Debug("WorkerPool: %d (for %s) has zero workers - adding %d temporary workers for %s", p.qid, mq.Name, boost, p.boostTimeout) - - start := time.Now() - pid := mq.RegisterWorkers(boost, start, true, start.Add(p.boostTimeout), cancel, false) - cancel = func() { - mq.RemoveWorkers(pid) - } - } else { - log.Debug("WorkerPool: %d has zero workers - adding %d temporary workers for %s", p.qid, p.boostWorkers, p.boostTimeout) - } - p.lock.Unlock() - p.addWorkers(ctx, cancel, boost) -} - -func (p *WorkerPool) pushBoost(data Data) { - select { - case p.dataChan <- data: - default: - p.lock.Lock() - if p.blockTimeout <= 0 { - p.lock.Unlock() - p.dataChan <- data - return - } - ourTimeout := p.blockTimeout - timer := time.NewTimer(p.blockTimeout) - p.lock.Unlock() - select { - case p.dataChan <- data: - util.StopTimer(timer) - case <-timer.C: - p.lock.Lock() - if p.blockTimeout > ourTimeout || (p.numberOfWorkers > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0) { - p.lock.Unlock() - p.dataChan <- data - return - } - p.blockTimeout *= 2 - boostCtx, boostCtxCancel := context.WithCancel(p.baseCtx) - mq := GetManager().GetManagedQueue(p.qid) - boost := p.boostWorkers - if (boost+p.numberOfWorkers) > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0 { - boost = p.maxNumberOfWorkers - p.numberOfWorkers - } - if mq != nil { - log.Debug("WorkerPool: %d (for %s) Channel blocked for %v - adding %d temporary workers for %s, block timeout now %v", p.qid, mq.Name, ourTimeout, boost, p.boostTimeout, p.blockTimeout) - - start := time.Now() - pid := mq.RegisterWorkers(boost, start, true, start.Add(p.boostTimeout), boostCtxCancel, false) - go func() { - <-boostCtx.Done() - mq.RemoveWorkers(pid) - boostCtxCancel() - }() - } else { - log.Debug("WorkerPool: %d Channel blocked for %v - adding %d temporary workers for %s, block timeout now %v", p.qid, ourTimeout, p.boostWorkers, p.boostTimeout, p.blockTimeout) - } - go func() { - <-time.After(p.boostTimeout) - boostCtxCancel() - p.lock.Lock() - p.blockTimeout /= 2 - p.lock.Unlock() - }() - p.lock.Unlock() - p.addWorkers(boostCtx, boostCtxCancel, boost) - p.dataChan <- data - } - } -} - -// NumberOfWorkers returns the number of current workers in the pool -func (p *WorkerPool) NumberOfWorkers() int { - p.lock.Lock() - defer p.lock.Unlock() - return p.numberOfWorkers -} - -// NumberInQueue returns the number of items in the queue -func (p *WorkerPool) NumberInQueue() int64 { - return atomic.LoadInt64(&p.numInQueue) -} - -// MaxNumberOfWorkers returns the maximum number of workers automatically added to the pool -func (p *WorkerPool) MaxNumberOfWorkers() int { - p.lock.Lock() - defer p.lock.Unlock() - return p.maxNumberOfWorkers -} - -// BoostWorkers returns the number of workers for a boost -func (p *WorkerPool) BoostWorkers() int { - p.lock.Lock() - defer p.lock.Unlock() - return p.boostWorkers -} - -// BoostTimeout returns the timeout of the next boost -func (p *WorkerPool) BoostTimeout() time.Duration { - p.lock.Lock() - defer p.lock.Unlock() - return p.boostTimeout -} - -// BlockTimeout returns the timeout til the next boost -func (p *WorkerPool) BlockTimeout() time.Duration { - p.lock.Lock() - defer p.lock.Unlock() - return p.blockTimeout -} - -// SetPoolSettings sets the setable boost values -func (p *WorkerPool) SetPoolSettings(maxNumberOfWorkers, boostWorkers int, timeout time.Duration) { - p.lock.Lock() - defer p.lock.Unlock() - p.maxNumberOfWorkers = maxNumberOfWorkers - p.boostWorkers = boostWorkers - p.boostTimeout = timeout -} - -// SetMaxNumberOfWorkers sets the maximum number of workers automatically added to the pool -// Changing this number will not change the number of current workers but will change the limit -// for future additions -func (p *WorkerPool) SetMaxNumberOfWorkers(newMax int) { - p.lock.Lock() - defer p.lock.Unlock() - p.maxNumberOfWorkers = newMax -} - -func (p *WorkerPool) commonRegisterWorkers(number int, timeout time.Duration, isFlusher bool) (context.Context, context.CancelFunc) { - var ctx context.Context - var cancel context.CancelFunc - start := time.Now() - end := start - hasTimeout := false - if timeout > 0 { - ctx, cancel = context.WithTimeout(p.baseCtx, timeout) - end = start.Add(timeout) - hasTimeout = true - } else { - ctx, cancel = context.WithCancel(p.baseCtx) - } - - mq := GetManager().GetManagedQueue(p.qid) - if mq != nil { - pid := mq.RegisterWorkers(number, start, hasTimeout, end, cancel, isFlusher) - log.Trace("WorkerPool: %d (for %s) adding %d workers with group id: %d", p.qid, mq.Name, number, pid) - return ctx, func() { - mq.RemoveWorkers(pid) - } - } - log.Trace("WorkerPool: %d adding %d workers (no group id)", p.qid, number) - - return ctx, cancel -} - -// AddWorkers adds workers to the pool - this allows the number of workers to go above the limit -func (p *WorkerPool) AddWorkers(number int, timeout time.Duration) context.CancelFunc { - ctx, cancel := p.commonRegisterWorkers(number, timeout, false) - p.addWorkers(ctx, cancel, number) - return cancel -} - -// addWorkers adds workers to the pool -func (p *WorkerPool) addWorkers(ctx context.Context, cancel context.CancelFunc, number int) { - for i := 0; i < number; i++ { - p.lock.Lock() - if p.cond == nil { - p.cond = sync.NewCond(&p.lock) - } - p.numberOfWorkers++ - p.lock.Unlock() - go func() { - pprof.SetGoroutineLabels(ctx) - p.doWork(ctx) - - p.lock.Lock() - p.numberOfWorkers-- - if p.numberOfWorkers == 0 { - p.cond.Broadcast() - cancel() - } else if p.numberOfWorkers < 0 { - // numberOfWorkers can't go negative but... - log.Warn("Number of Workers < 0 for QID %d - this shouldn't happen", p.qid) - p.numberOfWorkers = 0 - p.cond.Broadcast() - cancel() - } - select { - case <-p.baseCtx.Done(): - // Don't warn or check for ongoing work if the baseCtx is shutdown - case <-p.paused: - // Don't warn or check for ongoing work if the pool is paused - default: - if p.hasNoWorkerScaling() { - log.Warn( - "Queue: %d is configured to be non-scaling and has no workers - this configuration is likely incorrect.\n"+ - "The queue will be paused to prevent data-loss with the assumption that you will add workers and unpause as required.", p.qid) - p.pause() - } else if p.numberOfWorkers == 0 && atomic.LoadInt64(&p.numInQueue) > 0 { - // OK there are no workers but... there's still work to be done -> Reboost - p.zeroBoost() - // p.lock will be unlocked by zeroBoost - return - } - } - p.lock.Unlock() - }() - } -} - -// Wait for WorkerPool to finish -func (p *WorkerPool) Wait() { - p.lock.Lock() - defer p.lock.Unlock() - if p.cond == nil { - p.cond = sync.NewCond(&p.lock) - } - if p.numberOfWorkers <= 0 { - return - } - p.cond.Wait() -} - -// IsPaused returns if the pool is paused -func (p *WorkerPool) IsPaused() bool { - p.lock.Lock() - defer p.lock.Unlock() - select { - case <-p.paused: - return true - default: - return false - } -} - -// IsPausedIsResumed returns if the pool is paused and a channel that is closed when it is resumed -func (p *WorkerPool) IsPausedIsResumed() (<-chan struct{}, <-chan struct{}) { - p.lock.Lock() - defer p.lock.Unlock() - return p.paused, p.resumed -} - -// Pause pauses the WorkerPool -func (p *WorkerPool) Pause() { - p.lock.Lock() - defer p.lock.Unlock() - p.pause() -} - -func (p *WorkerPool) pause() { - select { - case <-p.paused: - default: - p.resumed = make(chan struct{}) - close(p.paused) - } -} - -// Resume resumes the WorkerPool -func (p *WorkerPool) Resume() { - p.lock.Lock() // can't defer unlock because of the zeroBoost at the end - select { - case <-p.resumed: - // already resumed - there's nothing to do - p.lock.Unlock() - return - default: - } - - p.paused = make(chan struct{}) - close(p.resumed) - - // OK now we need to check if we need to add some workers... - if p.numberOfWorkers > 0 || p.hasNoWorkerScaling() || atomic.LoadInt64(&p.numInQueue) == 0 { - // We either have workers, can't scale or there's no work to be done -> so just resume - p.lock.Unlock() - return - } - - // OK we got some work but no workers we need to think about boosting - select { - case <-p.baseCtx.Done(): - // don't bother boosting if the baseCtx is done - p.lock.Unlock() - return - default: - } - - // OK we'd better add some boost workers! - p.zeroBoost() - // p.zeroBoost will unlock the lock -} - -// CleanUp will drain the remaining contents of the channel -// This should be called after AddWorkers context is closed -func (p *WorkerPool) CleanUp(ctx context.Context) { - log.Trace("WorkerPool: %d CleanUp", p.qid) - close(p.dataChan) - for data := range p.dataChan { - if unhandled := p.handle(data); unhandled != nil { - if unhandled != nil { - log.Error("Unhandled Data in clean-up of queue %d", p.qid) - } - } - - atomic.AddInt64(&p.numInQueue, -1) - select { - case <-ctx.Done(): - log.Warn("WorkerPool: %d Cleanup context closed before finishing clean-up", p.qid) - return - default: - } - } - log.Trace("WorkerPool: %d CleanUp Done", p.qid) -} - -// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager -func (p *WorkerPool) Flush(timeout time.Duration) error { - ctx, cancel := p.commonRegisterWorkers(1, timeout, true) - defer cancel() - return p.FlushWithContext(ctx) -} - -// IsEmpty returns if true if the worker queue is empty -func (p *WorkerPool) IsEmpty() bool { - return atomic.LoadInt64(&p.numInQueue) == 0 -} - -// contextError returns either ctx.Done(), the base context's error or nil -func (p *WorkerPool) contextError(ctx context.Context) error { - select { - case <-p.baseCtx.Done(): - return p.baseCtx.Err() - case <-ctx.Done(): - return ctx.Err() - default: - return nil - } -} - -// FlushWithContext is very similar to CleanUp but it will return as soon as the dataChan is empty -// NB: The worker will not be registered with the manager. -func (p *WorkerPool) FlushWithContext(ctx context.Context) error { - log.Trace("WorkerPool: %d Flush", p.qid) - paused, _ := p.IsPausedIsResumed() - for { - // Because select will return any case that is satisified at random we precheck here before looking at dataChan. - select { - case <-paused: - // Ensure that even if paused that the cancelled error is still sent - return p.contextError(ctx) - case <-p.baseCtx.Done(): - return p.baseCtx.Err() - case <-ctx.Done(): - return ctx.Err() - default: - } - - select { - case <-paused: - return p.contextError(ctx) - case data, ok := <-p.dataChan: - if !ok { - return nil - } - if unhandled := p.handle(data); unhandled != nil { - log.Error("Unhandled Data whilst flushing queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1) - case <-p.baseCtx.Done(): - return p.baseCtx.Err() - case <-ctx.Done(): - return ctx.Err() - default: - return nil - } - } -} - -func (p *WorkerPool) doWork(ctx context.Context) { - pprof.SetGoroutineLabels(ctx) - delay := time.Millisecond * 300 - - // Create a common timer - we will use this elsewhere - timer := time.NewTimer(0) - util.StopTimer(timer) - - paused, _ := p.IsPausedIsResumed() - data := make([]Data, 0, p.batchLength) - for { - // Because select will return any case that is satisified at random we precheck here before looking at dataChan. - select { - case <-paused: - log.Trace("Worker for Queue %d Pausing", p.qid) - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - } - _, resumed := p.IsPausedIsResumed() - select { - case <-resumed: - paused, _ = p.IsPausedIsResumed() - log.Trace("Worker for Queue %d Resuming", p.qid) - util.StopTimer(timer) - case <-ctx.Done(): - log.Trace("Worker shutting down") - return - } - case <-ctx.Done(): - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - } - log.Trace("Worker shutting down") - return - default: - } - - select { - case <-paused: - // go back around - case <-ctx.Done(): - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - } - log.Trace("Worker shutting down") - return - case datum, ok := <-p.dataChan: - if !ok { - // the dataChan has been closed - we should finish up: - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - } - log.Trace("Worker shutting down") - return - } - data = append(data, datum) - util.StopTimer(timer) - - if len(data) >= p.batchLength { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - data = make([]Data, 0, p.batchLength) - } else { - timer.Reset(delay) - } - case <-timer.C: - delay = time.Millisecond * 100 - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - data = make([]Data, 0, p.batchLength) - } - } - } -} diff --git a/modules/queue/workerqueue.go b/modules/queue/workerqueue.go new file mode 100644 index 00000000000..de4485fa51e --- /dev/null +++ b/modules/queue/workerqueue.go @@ -0,0 +1,246 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +// WorkerPoolQueue is a queue that uses a pool of workers to process items +// It can use different underlying (base) queue types +type WorkerPoolQueue[T any] struct { + ctxRun context.Context + ctxRunCancel context.CancelFunc + ctxShutdown atomic.Pointer[context.Context] + shutdownDone chan struct{} + + origHandler HandlerFuncT[T] + safeHandler HandlerFuncT[T] + + baseQueueType string + baseConfig *BaseConfig + baseQueue baseQueue + + batchChan chan []T + flushChan chan flushType + + batchLength int + workerNum int + workerMaxNum int + workerActiveNum int + workerNumMu sync.Mutex +} + +type flushType chan struct{} + +var _ ManagedWorkerPoolQueue = (*WorkerPoolQueue[any])(nil) + +func (q *WorkerPoolQueue[T]) GetName() string { + return q.baseConfig.ManagedName +} + +func (q *WorkerPoolQueue[T]) GetType() string { + return q.baseQueueType +} + +func (q *WorkerPoolQueue[T]) GetItemTypeName() string { + var t T + return fmt.Sprintf("%T", t) +} + +func (q *WorkerPoolQueue[T]) GetWorkerNumber() int { + q.workerNumMu.Lock() + defer q.workerNumMu.Unlock() + return q.workerNum +} + +func (q *WorkerPoolQueue[T]) GetWorkerActiveNumber() int { + q.workerNumMu.Lock() + defer q.workerNumMu.Unlock() + return q.workerActiveNum +} + +func (q *WorkerPoolQueue[T]) GetWorkerMaxNumber() int { + q.workerNumMu.Lock() + defer q.workerNumMu.Unlock() + return q.workerMaxNum +} + +func (q *WorkerPoolQueue[T]) SetWorkerMaxNumber(num int) { + q.workerNumMu.Lock() + defer q.workerNumMu.Unlock() + q.workerMaxNum = num +} + +func (q *WorkerPoolQueue[T]) GetQueueItemNumber() int { + cnt, err := q.baseQueue.Len(q.ctxRun) + if err != nil { + log.Error("Failed to get number of items in queue %q: %v", q.GetName(), err) + } + return cnt +} + +func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.Duration) (err error) { + if q.isBaseQueueDummy() { + return + } + + log.Debug("Try to flush queue %q with timeout %v", q.GetName(), timeout) + defer log.Debug("Finish flushing queue %q, err: %v", q.GetName(), err) + + var after <-chan time.Time + after = infiniteTimerC + if timeout > 0 { + after = time.After(timeout) + } + c := make(flushType) + + // send flush request + // if it blocks, it means that there is a flush in progress or the queue hasn't been started yet + select { + case q.flushChan <- c: + case <-ctx.Done(): + return ctx.Err() + case <-q.ctxRun.Done(): + return q.ctxRun.Err() + case <-after: + return context.DeadlineExceeded + } + + // wait for flush to finish + select { + case <-c: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-q.ctxRun.Done(): + return q.ctxRun.Err() + case <-after: + return context.DeadlineExceeded + } +} + +// RemoveAllItems removes all items in the baes queue +func (q *WorkerPoolQueue[T]) RemoveAllItems(ctx context.Context) error { + return q.baseQueue.RemoveAll(ctx) +} + +func (q *WorkerPoolQueue[T]) marshal(data T) []byte { + bs, err := json.Marshal(data) + if err != nil { + log.Error("Failed to marshal item for queue %q: %v", q.GetName(), err) + return nil + } + return bs +} + +func (q *WorkerPoolQueue[T]) unmarshal(data []byte) (t T, ok bool) { + if err := json.Unmarshal(data, &t); err != nil { + log.Error("Failed to unmarshal item from queue %q: %v", q.GetName(), err) + return t, false + } + return t, true +} + +func (q *WorkerPoolQueue[T]) isBaseQueueDummy() bool { + _, isDummy := q.baseQueue.(*baseDummy) + return isDummy +} + +// Push adds an item to the queue, it may block for a while and then returns an error if the queue is full +func (q *WorkerPoolQueue[T]) Push(data T) error { + if q.isBaseQueueDummy() && q.safeHandler != nil { + // FIXME: the "immediate" queue is only for testing, but it really causes problems because its behavior is different from a real queue. + // Even if tests pass, it doesn't mean that there is no bug in code. + if data, ok := q.unmarshal(q.marshal(data)); ok { + q.safeHandler(data) + } + } + return q.baseQueue.PushItem(q.ctxRun, q.marshal(data)) +} + +// Has only works for unique queues. Keep in mind that this check may not be reliable (due to lacking of proper transaction support) +// There could be a small chance that duplicate items appear in the queue +func (q *WorkerPoolQueue[T]) Has(data T) (bool, error) { + return q.baseQueue.HasItem(q.ctxRun, q.marshal(data)) +} + +func (q *WorkerPoolQueue[T]) Run(atShutdown, atTerminate func(func())) { + atShutdown(func() { + // in case some queue handlers are slow or have hanging bugs, at most wait for a short time + q.ShutdownWait(1 * time.Second) + }) + q.doRun() +} + +// ShutdownWait shuts down the queue, waits for all workers to finish their jobs, and pushes the unhandled items back to the base queue +// It waits for all workers (handlers) to finish their jobs, in case some buggy handlers would hang forever, a reasonable timeout is needed +func (q *WorkerPoolQueue[T]) ShutdownWait(timeout time.Duration) { + shutdownCtx, shutdownCtxCancel := context.WithTimeout(context.Background(), timeout) + defer shutdownCtxCancel() + if q.ctxShutdown.CompareAndSwap(nil, &shutdownCtx) { + q.ctxRunCancel() + } + <-q.shutdownDone +} + +func getNewQueueFn(t string) (string, func(cfg *BaseConfig, unique bool) (baseQueue, error)) { + switch t { + case "dummy", "immediate": + return t, newBaseDummy + case "channel": + return t, newBaseChannelGeneric + case "redis": + return t, newBaseRedisGeneric + default: // level(leveldb,levelqueue,persistable-channel) + return "level", newBaseLevelQueueGeneric + } +} + +func NewWorkerPoolQueueBySetting[T any](name string, queueSetting setting.QueueSettings, handler HandlerFuncT[T], unique bool) (*WorkerPoolQueue[T], error) { + if handler == nil { + log.Debug("Use dummy queue for %q because handler is nil and caller doesn't want to process the queue items", name) + queueSetting.Type = "dummy" + } + + var w WorkerPoolQueue[T] + var err error + queueType, newQueueFn := getNewQueueFn(queueSetting.Type) + w.baseQueueType = queueType + w.baseConfig = toBaseConfig(name, queueSetting) + w.baseQueue, err = newQueueFn(w.baseConfig, unique) + if err != nil { + return nil, err + } + log.Trace("Created queue %q of type %q", name, queueType) + + w.ctxRun, w.ctxRunCancel = context.WithCancel(graceful.GetManager().ShutdownContext()) + w.batchChan = make(chan []T) + w.flushChan = make(chan flushType) + w.shutdownDone = make(chan struct{}) + w.workerMaxNum = queueSetting.MaxWorkers + w.batchLength = queueSetting.BatchLength + + w.origHandler = handler + w.safeHandler = func(t ...T) (unhandled []T) { + defer func() { + err := recover() + if err != nil { + log.Error("Recovered from panic in queue %q handler: %v\n%s", name, err, log.Stack(2)) + } + }() + return w.origHandler(t...) + } + + return &w, nil +} diff --git a/modules/queue/workerqueue_test.go b/modules/queue/workerqueue_test.go new file mode 100644 index 00000000000..da9451cd773 --- /dev/null +++ b/modules/queue/workerqueue_test.go @@ -0,0 +1,260 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "strconv" + "sync" + "testing" + "time" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func runWorkerPoolQueue[T any](q *WorkerPoolQueue[T]) func() { + var stop func() + started := make(chan struct{}) + stopped := make(chan struct{}) + go func() { + q.Run(func(f func()) { stop = f; close(started) }, nil) + close(stopped) + }() + <-started + return func() { + stop() + <-stopped + } +} + +func TestWorkerPoolQueueUnhandled(t *testing.T) { + oldUnhandledItemRequeueDuration := unhandledItemRequeueDuration.Load() + unhandledItemRequeueDuration.Store(0) + defer unhandledItemRequeueDuration.Store(oldUnhandledItemRequeueDuration) + + mu := sync.Mutex{} + + test := func(t *testing.T, queueSetting setting.QueueSettings) { + queueSetting.Length = 100 + queueSetting.Type = "channel" + queueSetting.Datadir = t.TempDir() + "/test-queue" + m := map[int]int{} + + // odds are handled once, evens are handled twice + handler := func(items ...int) (unhandled []int) { + testRecorder.Record("handle:%v", items) + for _, item := range items { + mu.Lock() + if item%2 == 0 && m[item] == 0 { + unhandled = append(unhandled, item) + } + m[item]++ + mu.Unlock() + } + return unhandled + } + + q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", queueSetting, handler, false) + stop := runWorkerPoolQueue(q) + for i := 0; i < queueSetting.Length; i++ { + testRecorder.Record("push:%v", i) + assert.NoError(t, q.Push(i)) + } + assert.NoError(t, q.FlushWithContext(context.Background(), 0)) + stop() + + ok := true + for i := 0; i < queueSetting.Length; i++ { + if i%2 == 0 { + ok = ok && assert.EqualValues(t, 2, m[i], "test %s: item %d", t.Name(), i) + } else { + ok = ok && assert.EqualValues(t, 1, m[i], "test %s: item %d", t.Name(), i) + } + } + if !ok { + t.Logf("m: %v", m) + t.Logf("records: %v", testRecorder.Records()) + } + testRecorder.Reset() + } + + runCount := 2 // we can run these tests even hundreds times to see its stability + t.Run("1/1", func(t *testing.T) { + for i := 0; i < runCount; i++ { + test(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1}) + } + }) + t.Run("3/1", func(t *testing.T) { + for i := 0; i < runCount; i++ { + test(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1}) + } + }) + t.Run("4/5", func(t *testing.T) { + for i := 0; i < runCount; i++ { + test(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5}) + } + }) +} + +func TestWorkerPoolQueuePersistence(t *testing.T) { + runCount := 2 // we can run these tests even hundreds times to see its stability + t.Run("1/1", func(t *testing.T) { + for i := 0; i < runCount; i++ { + testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1, Length: 100}) + } + }) + t.Run("3/1", func(t *testing.T) { + for i := 0; i < runCount; i++ { + testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1, Length: 100}) + } + }) + t.Run("4/5", func(t *testing.T) { + for i := 0; i < runCount; i++ { + testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5, Length: 100}) + } + }) +} + +func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSettings) { + testCount := queueSetting.Length + queueSetting.Type = "level" + queueSetting.Datadir = t.TempDir() + "/test-queue" + + mu := sync.Mutex{} + + var tasksQ1, tasksQ2 []string + q1 := func() { + startWhenAllReady := make(chan struct{}) // only start data consuming when the "testCount" tasks are all pushed into queue + stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item + + testHandler := func(data ...string) []string { + <-startWhenAllReady + time.Sleep(10 * time.Millisecond) + for _, s := range data { + mu.Lock() + tasksQ1 = append(tasksQ1, s) + mu.Unlock() + + if s == "task-20" { + close(stopAt20Shutdown) + } + } + return nil + } + + q, _ := NewWorkerPoolQueueBySetting("pr_patch_checker_test", queueSetting, testHandler, true) + stop := runWorkerPoolQueue(q) + for i := 0; i < testCount; i++ { + _ = q.Push("task-" + strconv.Itoa(i)) + } + close(startWhenAllReady) + <-stopAt20Shutdown // it's possible to have more than 20 tasks executed + stop() + } + + q1() // run some tasks and shutdown at an intermediate point + + time.Sleep(100 * time.Millisecond) // because the handler in q1 has a slight delay, we need to wait for it to finish + + q2 := func() { + testHandler := func(data ...string) []string { + for _, s := range data { + mu.Lock() + tasksQ2 = append(tasksQ2, s) + mu.Unlock() + } + return nil + } + + q, _ := NewWorkerPoolQueueBySetting("pr_patch_checker_test", queueSetting, testHandler, true) + stop := runWorkerPoolQueue(q) + assert.NoError(t, q.FlushWithContext(context.Background(), 0)) + stop() + } + + q2() // restart the queue to continue to execute the tasks in it + + assert.NotZero(t, len(tasksQ1)) + assert.NotZero(t, len(tasksQ2)) + assert.EqualValues(t, testCount, len(tasksQ1)+len(tasksQ2)) +} + +func TestWorkerPoolQueueActiveWorkers(t *testing.T) { + oldWorkerIdleDuration := workerIdleDuration + workerIdleDuration = 300 * time.Millisecond + defer func() { + workerIdleDuration = oldWorkerIdleDuration + }() + + handler := func(items ...int) (unhandled []int) { + time.Sleep(100 * time.Millisecond) + return nil + } + + q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 1, Length: 100}, handler, false) + stop := runWorkerPoolQueue(q) + for i := 0; i < 5; i++ { + assert.NoError(t, q.Push(i)) + } + + time.Sleep(50 * time.Millisecond) + assert.EqualValues(t, 1, q.GetWorkerNumber()) + assert.EqualValues(t, 1, q.GetWorkerActiveNumber()) + time.Sleep(500 * time.Millisecond) + assert.EqualValues(t, 1, q.GetWorkerNumber()) + assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + time.Sleep(workerIdleDuration) + assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + stop() + + q, _ = NewWorkerPoolQueueBySetting("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 3, Length: 100}, handler, false) + stop = runWorkerPoolQueue(q) + for i := 0; i < 15; i++ { + assert.NoError(t, q.Push(i)) + } + + time.Sleep(50 * time.Millisecond) + assert.EqualValues(t, 3, q.GetWorkerNumber()) + assert.EqualValues(t, 3, q.GetWorkerActiveNumber()) + time.Sleep(500 * time.Millisecond) + assert.EqualValues(t, 3, q.GetWorkerNumber()) + assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + time.Sleep(workerIdleDuration) + assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + stop() +} + +func TestWorkerPoolQueueShutdown(t *testing.T) { + oldUnhandledItemRequeueDuration := unhandledItemRequeueDuration.Load() + unhandledItemRequeueDuration.Store(int64(100 * time.Millisecond)) + defer unhandledItemRequeueDuration.Store(oldUnhandledItemRequeueDuration) + + // simulate a slow handler, it doesn't handle any item (all items will be pushed back to the queue) + handlerCalled := make(chan struct{}) + handler := func(items ...int) (unhandled []int) { + if items[0] == 0 { + close(handlerCalled) + } + time.Sleep(100 * time.Millisecond) + return items + } + + qs := setting.QueueSettings{Type: "level", Datadir: t.TempDir() + "/queue", BatchLength: 3, MaxWorkers: 4, Length: 20} + q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", qs, handler, false) + stop := runWorkerPoolQueue(q) + for i := 0; i < qs.Length; i++ { + assert.NoError(t, q.Push(i)) + } + <-handlerCalled + time.Sleep(50 * time.Millisecond) // wait for a while to make sure all workers are active + assert.EqualValues(t, 4, q.GetWorkerActiveNumber()) + stop() // stop triggers shutdown + assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + + // no item was ever handled, so we still get all of them again + q, _ = NewWorkerPoolQueueBySetting("test-workpoolqueue", qs, handler, false) + assert.EqualValues(t, 20, q.GetQueueItemNumber()) +} diff --git a/modules/repository/init.go b/modules/repository/init.go index cb353f24968..f079f72b771 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -195,9 +195,14 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, // LICENSE if len(opts.License) > 0 { - data, err = options.License(opts.License) + data, err = getLicense(opts.License, &licenseValues{ + Owner: repo.OwnerName, + Email: authorSig.Email, + Repo: repo.Name, + Year: time.Now().Format("2006"), + }) if err != nil { - return fmt.Errorf("GetRepoInitFile[%s]: %w", opts.License, err) + return fmt.Errorf("getLicense[%s]: %w", opts.License, err) } if err = os.WriteFile(filepath.Join(tmpDir, "LICENSE"), data, 0o644); err != nil { diff --git a/modules/repository/license.go b/modules/repository/license.go new file mode 100644 index 00000000000..5b188a041e2 --- /dev/null +++ b/modules/repository/license.go @@ -0,0 +1,113 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "bufio" + "bytes" + "fmt" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/options" +) + +type licenseValues struct { + Owner string + Email string + Repo string + Year string +} + +func getLicense(name string, values *licenseValues) ([]byte, error) { + data, err := options.License(name) + if err != nil { + return nil, fmt.Errorf("GetRepoInitFile[%s]: %w", name, err) + } + return fillLicensePlaceholder(name, values, data), nil +} + +func fillLicensePlaceholder(name string, values *licenseValues, origin []byte) []byte { + placeholder := getLicensePlaceholder(name) + + scanner := bufio.NewScanner(bytes.NewReader(origin)) + output := bytes.NewBuffer(nil) + for scanner.Scan() { + line := scanner.Text() + if placeholder.MatchLine == nil || placeholder.MatchLine.MatchString(line) { + for _, v := range placeholder.Owner { + line = strings.ReplaceAll(line, v, values.Owner) + } + for _, v := range placeholder.Email { + line = strings.ReplaceAll(line, v, values.Email) + } + for _, v := range placeholder.Repo { + line = strings.ReplaceAll(line, v, values.Repo) + } + for _, v := range placeholder.Year { + line = strings.ReplaceAll(line, v, values.Year) + } + } + output.WriteString(line + "\n") + } + + return output.Bytes() +} + +type licensePlaceholder struct { + Owner []string + Email []string + Repo []string + Year []string + MatchLine *regexp.Regexp +} + +func getLicensePlaceholder(name string) *licensePlaceholder { + // Some universal placeholders. + // If you want to add a new one, make sure you have check it by `grep -r 'NEW_WORD' options/license` and all of them are placeholders. + ret := &licensePlaceholder{ + Owner: []string{ + "", + "", + "[NAME]", + "[name of copyright owner]", + "[name of copyright holder]", + "", + "", + "", + "", + "[one or more legally recognised persons or entities offering the Work under the terms and conditions of this Licence]", + }, + Email: []string{ + "[EMAIL]", + }, + Repo: []string{ + "", + "", + }, + Year: []string{ + "", + "[YEAR]", + "{YEAR}", + "[yyyy]", + "[Year]", + "[year]", + }, + } + + // Some special placeholders for specific licenses. + // It's unsafe to apply them to all licenses. + switch name { + case "0BSD": + return &licensePlaceholder{ + Owner: []string{"AUTHOR"}, + Email: []string{"EMAIL"}, + Year: []string{"YEAR"}, + MatchLine: regexp.MustCompile(`Copyright \(C\) YEAR by AUTHOR EMAIL`), // there is another AUTHOR in the file, but it's not a placeholder + } + + // Other special placeholders can be added here. + } + return ret +} diff --git a/modules/repository/license_test.go b/modules/repository/license_test.go new file mode 100644 index 00000000000..13c865693c3 --- /dev/null +++ b/modules/repository/license_test.go @@ -0,0 +1,181 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_getLicense(t *testing.T) { + type args struct { + name string + values *licenseValues + } + tests := []struct { + name string + args args + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "regular", + args: args{ + name: "MIT", + values: &licenseValues{Owner: "Gitea", Year: "2023"}, + }, + want: `MIT License + +Copyright (c) 2023 Gitea + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +`, + wantErr: assert.NoError, + }, + { + name: "license not found", + args: args{ + name: "notfound", + }, + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getLicense(tt.args.name, tt.args.values) + if !tt.wantErr(t, err, fmt.Sprintf("getLicense(%v, %v)", tt.args.name, tt.args.values)) { + return + } + assert.Equalf(t, tt.want, string(got), "getLicense(%v, %v)", tt.args.name, tt.args.values) + }) + } +} + +func Test_fillLicensePlaceholder(t *testing.T) { + type args struct { + name string + values *licenseValues + origin string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "owner", + args: args{ + name: "regular", + values: &licenseValues{Year: "2023", Owner: "Gitea", Email: "teabot@gitea.io", Repo: "gitea"}, + origin: ` + + +[NAME] +[name of copyright owner] +[name of copyright holder] + + + + +[one or more legally recognised persons or entities offering the Work under the terms and conditions of this Licence] +`, + }, + want: ` +Gitea +Gitea +Gitea +Gitea +Gitea +Gitea +Gitea +Gitea +Gitea +Gitea +`, + }, + { + name: "email", + args: args{ + name: "regular", + values: &licenseValues{Year: "2023", Owner: "Gitea", Email: "teabot@gitea.io", Repo: "gitea"}, + origin: ` +[EMAIL] +`, + }, + want: ` +teabot@gitea.io +`, + }, + { + name: "repo", + args: args{ + name: "regular", + values: &licenseValues{Year: "2023", Owner: "Gitea", Email: "teabot@gitea.io", Repo: "gitea"}, + origin: ` + + +`, + }, + want: ` +gitea +gitea +`, + }, + { + name: "year", + args: args{ + name: "regular", + values: &licenseValues{Year: "2023", Owner: "Gitea", Email: "teabot@gitea.io", Repo: "gitea"}, + origin: ` + +[YEAR] +{YEAR} +[yyyy] +[Year] +[year] +`, + }, + want: ` +2023 +2023 +2023 +2023 +2023 +2023 +`, + }, + { + name: "0BSD", + args: args{ + name: "0BSD", + values: &licenseValues{Year: "2023", Owner: "Gitea", Email: "teabot@gitea.io", Repo: "gitea"}, + origin: ` +Copyright (C) YEAR by AUTHOR EMAIL + +... + +... THE AUTHOR BE LIABLE FOR ... +`, + }, + want: ` +Copyright (C) 2023 by Gitea teabot@gitea.io + +... + +... THE AUTHOR BE LIABLE FOR ... +`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, string(fillLicensePlaceholder(tt.args.name, tt.args.values, []byte(tt.args.origin))), "fillLicensePlaceholder(%v, %v, %v)", tt.args.name, tt.args.values, tt.args.origin) + }) + } +} diff --git a/modules/secret/secret.go b/modules/secret/secret.go index 628ae505a5c..9c2ecd181d2 100644 --- a/modules/secret/secret.go +++ b/modules/secret/secret.go @@ -10,6 +10,7 @@ import ( "encoding/base64" "encoding/hex" "errors" + "fmt" "io" "github.com/minio/sha256-simd" @@ -19,13 +20,13 @@ import ( func AesEncrypt(key, text []byte) ([]byte, error) { block, err := aes.NewCipher(key) if err != nil { - return nil, err + return nil, fmt.Errorf("AesEncrypt invalid key: %v", err) } b := base64.StdEncoding.EncodeToString(text) ciphertext := make([]byte, aes.BlockSize+len(b)) iv := ciphertext[:aes.BlockSize] - if _, err := io.ReadFull(rand.Reader, iv); err != nil { - return nil, err + if _, err = io.ReadFull(rand.Reader, iv); err != nil { + return nil, fmt.Errorf("AesEncrypt unable to read IV: %w", err) } cfb := cipher.NewCFBEncrypter(block, iv) cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b)) @@ -39,7 +40,7 @@ func AesDecrypt(key, text []byte) ([]byte, error) { return nil, err } if len(text) < aes.BlockSize { - return nil, errors.New("ciphertext too short") + return nil, errors.New("AesDecrypt ciphertext too short") } iv := text[:aes.BlockSize] text = text[aes.BlockSize:] @@ -47,7 +48,7 @@ func AesDecrypt(key, text []byte) ([]byte, error) { cfb.XORKeyStream(text, text) data, err := base64.StdEncoding.DecodeString(string(text)) if err != nil { - return nil, err + return nil, fmt.Errorf("AesDecrypt invalid decrypted base64 string: %w", err) } return data, nil } @@ -58,21 +59,21 @@ func EncryptSecret(key, str string) (string, error) { plaintext := []byte(str) ciphertext, err := AesEncrypt(keyHash[:], plaintext) if err != nil { - return "", err + return "", fmt.Errorf("failed to encrypt by secret: %w", err) } return hex.EncodeToString(ciphertext), nil } // DecryptSecret decrypts a previously encrypted hex string -func DecryptSecret(key, cipherhex string) (string, error) { +func DecryptSecret(key, cipherHex string) (string, error) { keyHash := sha256.Sum256([]byte(key)) - ciphertext, err := hex.DecodeString(cipherhex) + ciphertext, err := hex.DecodeString(cipherHex) if err != nil { - return "", err + return "", fmt.Errorf("failed to decrypt by secret, invalid hex string: %w", err) } plaintext, err := AesDecrypt(keyHash[:], ciphertext) if err != nil { - return "", err + return "", fmt.Errorf("failed to decrypt by secret, the key (maybe SECRET_KEY?) might be incorrect: %w", err) } return string(plaintext), nil } diff --git a/modules/secret/secret_test.go b/modules/secret/secret_test.go index 4b018fddb67..d4fb46955b7 100644 --- a/modules/secret/secret_test.go +++ b/modules/secret/secret_test.go @@ -10,14 +10,22 @@ import ( ) func TestEncryptDecrypt(t *testing.T) { - var hex string - var str string - - hex, _ = EncryptSecret("foo", "baz") - str, _ = DecryptSecret("foo", hex) + hex, err := EncryptSecret("foo", "baz") + assert.NoError(t, err) + str, _ := DecryptSecret("foo", hex) assert.Equal(t, "baz", str) - hex, _ = EncryptSecret("bar", "baz") + hex, err = EncryptSecret("bar", "baz") + assert.NoError(t, err) str, _ = DecryptSecret("foo", hex) assert.NotEqual(t, "baz", str) + + _, err = DecryptSecret("a", "b") + assert.ErrorContains(t, err, "invalid hex string") + + _, err = DecryptSecret("a", "bb") + assert.ErrorContains(t, err, "the key (maybe SECRET_KEY?) might be incorrect: AesDecrypt ciphertext too short") + + _, err = DecryptSecret("a", "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + assert.ErrorContains(t, err, "the key (maybe SECRET_KEY?) might be incorrect: AesDecrypt invalid decrypted base64 string") } diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 92c8c97fe95..ce9ef72248a 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -30,25 +30,17 @@ type ConfigProvider interface { Section(section string) ConfigSection NewSection(name string) (ConfigSection, error) GetSection(name string) (ConfigSection, error) - DeleteSection(name string) error Save() error } type iniFileConfigProvider struct { + opts *Options *ini.File - filepath string // the ini file path - newFile bool // whether the file has not existed previously - allowEmpty bool // whether not finding configuration files is allowed (only true for the tests) + newFile bool // whether the file has not existed previously } -// NewEmptyConfigProvider create a new empty config provider -func NewEmptyConfigProvider() ConfigProvider { - cp, _ := newConfigProviderFromData("") - return cp -} - -// newConfigProviderFromData this function is only for testing -func newConfigProviderFromData(configContent string) (ConfigProvider, error) { +// NewConfigProviderFromData this function is only for testing +func NewConfigProviderFromData(configContent string) (ConfigProvider, error) { var cfg *ini.File var err error if configContent == "" { @@ -66,41 +58,47 @@ func newConfigProviderFromData(configContent string) (ConfigProvider, error) { }, nil } +type Options struct { + CustomConf string // the ini file path + AllowEmpty bool // whether not finding configuration files is allowed (only true for the tests) + ExtraConfig string + DisableLoadCommonSettings bool +} + // newConfigProviderFromFile load configuration from file. // NOTE: do not print any log except error. -func newConfigProviderFromFile(customConf string, allowEmpty bool, extraConfig string) (*iniFileConfigProvider, error) { +func newConfigProviderFromFile(opts *Options) (*iniFileConfigProvider, error) { cfg := ini.Empty() newFile := true - if customConf != "" { - isFile, err := util.IsFile(customConf) + if opts.CustomConf != "" { + isFile, err := util.IsFile(opts.CustomConf) if err != nil { - return nil, fmt.Errorf("unable to check if %s is a file. Error: %v", customConf, err) + return nil, fmt.Errorf("unable to check if %s is a file. Error: %v", opts.CustomConf, err) } if isFile { - if err := cfg.Append(customConf); err != nil { - return nil, fmt.Errorf("failed to load custom conf '%s': %v", customConf, err) + if err := cfg.Append(opts.CustomConf); err != nil { + return nil, fmt.Errorf("failed to load custom conf '%s': %v", opts.CustomConf, err) } newFile = false } } - if newFile && !allowEmpty { + if newFile && !opts.AllowEmpty { return nil, fmt.Errorf("unable to find configuration file: %q, please ensure you are running in the correct environment or set the correct configuration file with -c", CustomConf) } - if extraConfig != "" { - if err := cfg.Append([]byte(extraConfig)); err != nil { + if opts.ExtraConfig != "" { + if err := cfg.Append([]byte(opts.ExtraConfig)); err != nil { return nil, fmt.Errorf("unable to append more config: %v", err) } } cfg.NameMapper = ini.SnackCase return &iniFileConfigProvider{ - File: cfg, - filepath: customConf, - newFile: newFile, - allowEmpty: allowEmpty, + opts: opts, + File: cfg, + newFile: newFile, }, nil } @@ -116,15 +114,10 @@ func (p *iniFileConfigProvider) GetSection(name string) (ConfigSection, error) { return p.File.GetSection(name) } -func (p *iniFileConfigProvider) DeleteSection(name string) error { - p.File.DeleteSection(name) - return nil -} - // Save save the content into file func (p *iniFileConfigProvider) Save() error { - if p.filepath == "" { - if !p.allowEmpty { + if p.opts.CustomConf == "" { + if !p.opts.AllowEmpty { return fmt.Errorf("custom config path must not be empty") } return nil @@ -135,8 +128,8 @@ func (p *iniFileConfigProvider) Save() error { return fmt.Errorf("failed to create '%s': %v", CustomConf, err) } } - if err := p.SaveTo(p.filepath); err != nil { - return fmt.Errorf("failed to save '%s': %v", p.filepath, err) + if err := p.SaveTo(p.opts.CustomConf); err != nil { + return fmt.Errorf("failed to save '%s': %v", p.opts.CustomConf, err) } // Change permissions to be more restrictive diff --git a/modules/setting/cron_test.go b/modules/setting/cron_test.go index 8d58cf8b486..3187ab18a25 100644 --- a/modules/setting/cron_test.go +++ b/modules/setting/cron_test.go @@ -26,7 +26,7 @@ BASE = true SECOND = white rabbit EXTEND = true ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) extended := &Extended{ diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 8aee8596ded..6836e623112 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -70,15 +70,6 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) - // The following settings are deprecated and can be overridden by settings in [queue] or [queue.issue_indexer] - // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version - // if these are removed, the warning will not be shown - deprecatedSetting(rootCfg, "indexer", "ISSUE_INDEXER_QUEUE_TYPE", "queue.issue_indexer", "TYPE", "v1.19.0") - deprecatedSetting(rootCfg, "indexer", "ISSUE_INDEXER_QUEUE_DIR", "queue.issue_indexer", "DATADIR", "v1.19.0") - deprecatedSetting(rootCfg, "indexer", "ISSUE_INDEXER_QUEUE_CONN_STR", "queue.issue_indexer", "CONN_STR", "v1.19.0") - deprecatedSetting(rootCfg, "indexer", "ISSUE_INDEXER_QUEUE_BATCH_NUMBER", "queue.issue_indexer", "BATCH_LENGTH", "v1.19.0") - deprecatedSetting(rootCfg, "indexer", "UPDATE_BUFFER_LEN", "queue.issue_indexer", "LENGTH", "v1.19.0") - Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) diff --git a/modules/setting/lfs.go b/modules/setting/lfs.go index 1b659dd22b0..a5c3631681b 100644 --- a/modules/setting/lfs.go +++ b/modules/setting/lfs.go @@ -45,7 +45,7 @@ func loadLFSFrom(rootCfg ConfigProvider) { LFS.LocksPagingNum = 50 } - LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(20 * time.Minute) + LFS.HTTPAuthExpiry = sec.Key("LFS_HTTP_AUTH_EXPIRY").MustDuration(24 * time.Hour) if LFS.StartServer { LFS.JWTSecretBytes = make([]byte, 32) diff --git a/modules/setting/mailer_test.go b/modules/setting/mailer_test.go index f65dbaf8f57..fbabf113786 100644 --- a/modules/setting/mailer_test.go +++ b/modules/setting/mailer_test.go @@ -10,7 +10,6 @@ import ( ) func Test_loadMailerFrom(t *testing.T) { - iniFile := NewEmptyConfigProvider() kases := map[string]*Mailer{ "smtp.mydomain.com": { SMTPAddr: "smtp.mydomain.com", @@ -27,13 +26,13 @@ func Test_loadMailerFrom(t *testing.T) { } for host, kase := range kases { t.Run(host, func(t *testing.T) { - iniFile.DeleteSection("mailer") - sec := iniFile.Section("mailer") + cfg, _ := NewConfigProviderFromData("") + sec := cfg.Section("mailer") sec.NewKey("ENABLED", "true") sec.NewKey("HOST", host) // Check mailer setting - loadMailerFrom(iniFile) + loadMailerFrom(cfg) assert.EqualValues(t, kase.SMTPAddr, MailService.SMTPAddr) assert.EqualValues(t, kase.SMTPPort, MailService.SMTPPort) diff --git a/modules/setting/packages.go b/modules/setting/packages.go index b52bbf40c7b..00d8b6122f3 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -38,6 +38,7 @@ var ( LimitSizeNuGet int64 LimitSizePub int64 LimitSizePyPI int64 + LimitSizeRpm int64 LimitSizeRubyGems int64 LimitSizeSwift int64 LimitSizeVagrant int64 @@ -82,6 +83,7 @@ func loadPackagesFrom(rootCfg ConfigProvider) { Packages.LimitSizeNuGet = mustBytes(sec, "LIMIT_SIZE_NUGET") Packages.LimitSizePub = mustBytes(sec, "LIMIT_SIZE_PUB") Packages.LimitSizePyPI = mustBytes(sec, "LIMIT_SIZE_PYPI") + Packages.LimitSizeRpm = mustBytes(sec, "LIMIT_SIZE_RPM") Packages.LimitSizeRubyGems = mustBytes(sec, "LIMIT_SIZE_RUBYGEMS") Packages.LimitSizeSwift = mustBytes(sec, "LIMIT_SIZE_SWIFT") Packages.LimitSizeVagrant = mustBytes(sec, "LIMIT_SIZE_VAGRANT") diff --git a/modules/setting/packages_test.go b/modules/setting/packages_test.go index 04d6e558346..d9f6e105287 100644 --- a/modules/setting/packages_test.go +++ b/modules/setting/packages_test.go @@ -7,12 +7,13 @@ import ( "testing" "github.com/stretchr/testify/assert" - ini "gopkg.in/ini.v1" ) func TestMustBytes(t *testing.T) { test := func(value string) int64 { - sec, _ := ini.Empty().NewSection("test") + cfg, err := NewConfigProviderFromData("[test]") + assert.NoError(t, err) + sec := cfg.Section("test") sec.NewKey("VALUE", value) return mustBytes(sec, "VALUE") diff --git a/modules/setting/queue.go b/modules/setting/queue.go index 8c37e538bbe..8673537b525 100644 --- a/modules/setting/queue.go +++ b/modules/setting/queue.go @@ -5,198 +5,109 @@ package setting import ( "path/filepath" - "strconv" - "time" - "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" ) // QueueSettings represent the settings for a queue from the ini type QueueSettings struct { - Name string - DataDir string - QueueLength int `ini:"LENGTH"` - BatchLength int - ConnectionString string - Type string - QueueName string - SetName string - WrapIfNecessary bool - MaxAttempts int - Timeout time.Duration - Workers int - MaxWorkers int - BlockTimeout time.Duration - BoostTimeout time.Duration - BoostWorkers int + Name string // not an INI option, it is the name for [queue.the-name] section + + Type string + Datadir string + ConnStr string // for leveldb or redis + Length int // max queue length before blocking + + QueueName, SetName string // the name suffix for storage (db key, redis key), "set" is for unique queue + + BatchLength int + MaxWorkers int } -// Queue settings -var Queue = QueueSettings{} +var queueSettingsDefault = QueueSettings{ + Type: "level", // dummy, channel, level, redis + Datadir: "queues/common", // relative to AppDataPath + Length: 100, // queue length before a channel queue will block -// GetQueueSettings returns the queue settings for the appropriately named queue -func GetQueueSettings(name string) QueueSettings { - return getQueueSettings(CfgProvider, name) + QueueName: "_queue", + SetName: "_unique", + BatchLength: 20, + MaxWorkers: 10, } -func getQueueSettings(rootCfg ConfigProvider, name string) QueueSettings { - q := QueueSettings{} - sec := rootCfg.Section("queue." + name) - q.Name = name +func GetQueueSettings(rootCfg ConfigProvider, name string) (QueueSettings, error) { + // deep copy default settings + cfg := QueueSettings{} + if cfgBs, err := json.Marshal(queueSettingsDefault); err != nil { + return cfg, err + } else if err = json.Unmarshal(cfgBs, &cfg); err != nil { + return cfg, err + } - // DataDir is not directly inheritable - q.DataDir = filepath.ToSlash(filepath.Join(Queue.DataDir, "common")) - // QueueName is not directly inheritable either - q.QueueName = name + Queue.QueueName - for _, key := range sec.Keys() { - switch key.Name() { - case "DATADIR": - q.DataDir = key.MustString(q.DataDir) - case "QUEUE_NAME": - q.QueueName = key.MustString(q.QueueName) - case "SET_NAME": - q.SetName = key.MustString(q.SetName) + cfg.Name = name + if sec, err := rootCfg.GetSection("queue"); err == nil { + if err = sec.MapTo(&cfg); err != nil { + log.Error("Failed to map queue common config for %q: %v", name, err) + return cfg, nil } } - if len(q.SetName) == 0 && len(Queue.SetName) > 0 { - q.SetName = q.QueueName + Queue.SetName + if sec, err := rootCfg.GetSection("queue." + name); err == nil { + if err = sec.MapTo(&cfg); err != nil { + log.Error("Failed to map queue spec config for %q: %v", name, err) + return cfg, nil + } + if sec.HasKey("CONN_STR") { + cfg.ConnStr = sec.Key("CONN_STR").String() + } } - if !filepath.IsAbs(q.DataDir) { - q.DataDir = filepath.ToSlash(filepath.Join(AppDataPath, q.DataDir)) + + if cfg.Datadir == "" { + cfg.Datadir = queueSettingsDefault.Datadir } - _, _ = sec.NewKey("DATADIR", q.DataDir) + if !filepath.IsAbs(cfg.Datadir) { + cfg.Datadir = filepath.Join(AppDataPath, cfg.Datadir) + } + cfg.Datadir = filepath.ToSlash(cfg.Datadir) - // The rest are... - q.QueueLength = sec.Key("LENGTH").MustInt(Queue.QueueLength) - q.BatchLength = sec.Key("BATCH_LENGTH").MustInt(Queue.BatchLength) - q.ConnectionString = sec.Key("CONN_STR").MustString(Queue.ConnectionString) - q.Type = sec.Key("TYPE").MustString(Queue.Type) - q.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(Queue.WrapIfNecessary) - q.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Queue.MaxAttempts) - q.Timeout = sec.Key("TIMEOUT").MustDuration(Queue.Timeout) - q.Workers = sec.Key("WORKERS").MustInt(Queue.Workers) - q.MaxWorkers = sec.Key("MAX_WORKERS").MustInt(Queue.MaxWorkers) - q.BlockTimeout = sec.Key("BLOCK_TIMEOUT").MustDuration(Queue.BlockTimeout) - q.BoostTimeout = sec.Key("BOOST_TIMEOUT").MustDuration(Queue.BoostTimeout) - q.BoostWorkers = sec.Key("BOOST_WORKERS").MustInt(Queue.BoostWorkers) + if cfg.Type == "redis" && cfg.ConnStr == "" { + cfg.ConnStr = "redis://127.0.0.1:6379/0" + } - return q + if cfg.Length <= 0 { + cfg.Length = queueSettingsDefault.Length + } + if cfg.MaxWorkers <= 0 { + cfg.MaxWorkers = queueSettingsDefault.MaxWorkers + } + if cfg.BatchLength <= 0 { + cfg.BatchLength = queueSettingsDefault.BatchLength + } + + return cfg, nil } -// LoadQueueSettings sets up the default settings for Queues -// This is exported for tests to be able to use the queue func LoadQueueSettings() { loadQueueFrom(CfgProvider) } func loadQueueFrom(rootCfg ConfigProvider) { - sec := rootCfg.Section("queue") - Queue.DataDir = filepath.ToSlash(sec.Key("DATADIR").MustString("queues/")) - if !filepath.IsAbs(Queue.DataDir) { - Queue.DataDir = filepath.ToSlash(filepath.Join(AppDataPath, Queue.DataDir)) - } - Queue.QueueLength = sec.Key("LENGTH").MustInt(20) - Queue.BatchLength = sec.Key("BATCH_LENGTH").MustInt(20) - Queue.ConnectionString = sec.Key("CONN_STR").MustString("") - defaultType := sec.Key("TYPE").String() - Queue.Type = sec.Key("TYPE").MustString("persistable-channel") - Queue.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(true) - Queue.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(10) - Queue.Timeout = sec.Key("TIMEOUT").MustDuration(GracefulHammerTime + 30*time.Second) - Queue.Workers = sec.Key("WORKERS").MustInt(0) - Queue.MaxWorkers = sec.Key("MAX_WORKERS").MustInt(10) - Queue.BlockTimeout = sec.Key("BLOCK_TIMEOUT").MustDuration(1 * time.Second) - Queue.BoostTimeout = sec.Key("BOOST_TIMEOUT").MustDuration(5 * time.Minute) - Queue.BoostWorkers = sec.Key("BOOST_WORKERS").MustInt(1) - Queue.QueueName = sec.Key("QUEUE_NAME").MustString("_queue") - Queue.SetName = sec.Key("SET_NAME").MustString("") - - // Now handle the old issue_indexer configuration - // FIXME: DEPRECATED to be removed in v1.18.0 - section := rootCfg.Section("queue.issue_indexer") - directlySet := toDirectlySetKeysSet(section) - if !directlySet.Contains("TYPE") && defaultType == "" { - switch typ := rootCfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(""); typ { - case "levelqueue": - _, _ = section.NewKey("TYPE", "level") - case "channel": - _, _ = section.NewKey("TYPE", "persistable-channel") - case "redis": - _, _ = section.NewKey("TYPE", "redis") - case "": - _, _ = section.NewKey("TYPE", "level") - default: - log.Fatal("Unsupported indexer queue type: %v", typ) + hasOld := false + handleOldLengthConfiguration := func(rootCfg ConfigProvider, newQueueName, oldSection, oldKey string) { + if rootCfg.Section(oldSection).HasKey(oldKey) { + hasOld = true + log.Error("Removed queue option: `[%s].%s`. Use new options in `[queue.%s]`", oldSection, oldKey, newQueueName) } } - if !directlySet.Contains("LENGTH") { - length := rootCfg.Section("indexer").Key("UPDATE_BUFFER_LEN").MustInt(0) - if length != 0 { - _, _ = section.NewKey("LENGTH", strconv.Itoa(length)) - } - } - if !directlySet.Contains("BATCH_LENGTH") { - fallback := rootCfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(0) - if fallback != 0 { - _, _ = section.NewKey("BATCH_LENGTH", strconv.Itoa(fallback)) - } - } - if !directlySet.Contains("DATADIR") { - queueDir := filepath.ToSlash(rootCfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_DIR").MustString("")) - if queueDir != "" { - _, _ = section.NewKey("DATADIR", queueDir) - } - } - if !directlySet.Contains("CONN_STR") { - connStr := rootCfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString("") - if connStr != "" { - _, _ = section.NewKey("CONN_STR", connStr) - } - } - - // FIXME: DEPRECATED to be removed in v1.18.0 - // - will need to set default for [queue.*)] LENGTH appropriately though though - - // Handle the old mailer configuration - handleOldLengthConfiguration(rootCfg, "mailer", "mailer", "SEND_BUFFER_LEN", 100) - - // Handle the old test pull requests configuration - // Please note this will be a unique queue - handleOldLengthConfiguration(rootCfg, "pr_patch_checker", "repository", "PULL_REQUEST_QUEUE_LENGTH", 1000) - - // Handle the old mirror queue configuration - // Please note this will be a unique queue - handleOldLengthConfiguration(rootCfg, "mirror", "repository", "MIRROR_QUEUE_LENGTH", 1000) -} - -// handleOldLengthConfiguration allows fallback to older configuration. `[queue.name]` `LENGTH` will override this configuration, but -// if that is left unset then we should fallback to the older configuration. (Except where the new length woul be <=0) -func handleOldLengthConfiguration(rootCfg ConfigProvider, queueName, oldSection, oldKey string, defaultValue int) { - if rootCfg.Section(oldSection).HasKey(oldKey) { - log.Error("Deprecated fallback for %s queue length `[%s]` `%s` present. Use `[queue.%s]` `LENGTH`. This will be removed in v1.18.0", queueName, queueName, oldSection, oldKey) - } - value := rootCfg.Section(oldSection).Key(oldKey).MustInt(defaultValue) - - // Don't override with 0 - if value <= 0 { - return - } - - section := rootCfg.Section("queue." + queueName) - directlySet := toDirectlySetKeysSet(section) - if !directlySet.Contains("LENGTH") { - _, _ = section.NewKey("LENGTH", strconv.Itoa(value)) + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "ISSUE_INDEXER_QUEUE_TYPE") + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "ISSUE_INDEXER_QUEUE_BATCH_NUMBER") + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "ISSUE_INDEXER_QUEUE_DIR") + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "ISSUE_INDEXER_QUEUE_CONN_STR") + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "UPDATE_BUFFER_LEN") + handleOldLengthConfiguration(rootCfg, "mailer", "mailer", "SEND_BUFFER_LEN") + handleOldLengthConfiguration(rootCfg, "pr_patch_checker", "repository", "PULL_REQUEST_QUEUE_LENGTH") + handleOldLengthConfiguration(rootCfg, "mirror", "repository", "MIRROR_QUEUE_LENGTH") + if hasOld { + log.Fatal("Please update your app.ini to remove deprecated config options") } } - -// toDirectlySetKeysSet returns a set of keys directly set by this section -// Note: we cannot use section.HasKey(...) as that will immediately set the Key if a parent section has the Key -// but this section does not. -func toDirectlySetKeysSet(section ConfigSection) container.Set[string] { - sections := make(container.Set[string]) - for _, key := range section.Keys() { - sections.Add(key.Name()) - } - return sections -} diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 20ed6d0dcd5..56e7e6f4aca 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -308,6 +308,10 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { Repository.DisabledRepoUnits = append(Repository.DisabledRepoUnits, "repo.packages") } + if !rootCfg.Section("actions").Key("ENABLED").MustBool(true) { + Repository.DisabledRepoUnits = append(Repository.DisabledRepoUnits, "repo.actions") + } + // Handle default trustmodel settings Repository.Signing.DefaultTrustModel = strings.ToLower(strings.TrimSpace(Repository.Signing.DefaultTrustModel)) if Repository.Signing.DefaultTrustModel == "default" { diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 9ab55e91c53..b085a7b3214 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -15,7 +15,6 @@ import ( "strings" "time" - "code.gitea.io/gitea/modules/auth/password/hash" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/user" ) @@ -203,46 +202,20 @@ func PrepareAppDataPath() error { return nil } -// InitProviderFromExistingFile initializes config provider from an existing config file (app.ini) -func InitProviderFromExistingFile() { +func Init(opts *Options) { + if opts.CustomConf == "" { + opts.CustomConf = CustomConf + } var err error - CfgProvider, err = newConfigProviderFromFile(CustomConf, false, "") + CfgProvider, err = newConfigProviderFromFile(opts) if err != nil { - log.Fatal("InitProviderFromExistingFile: %v", err) + log.Fatal("Init[%v]: %v", opts, err) } -} - -// InitProviderAllowEmpty initializes config provider from file, it's also fine that if the config file (app.ini) doesn't exist -func InitProviderAllowEmpty() { - var err error - CfgProvider, err = newConfigProviderFromFile(CustomConf, true, "") - if err != nil { - log.Fatal("InitProviderAllowEmpty: %v", err) + if !opts.DisableLoadCommonSettings { + loadCommonSettingsFrom(CfgProvider) } } -// InitProviderAndLoadCommonSettingsForTest initializes config provider and load common setttings for tests -func InitProviderAndLoadCommonSettingsForTest(extraConfigs ...string) { - var err error - CfgProvider, err = newConfigProviderFromFile(CustomConf, true, strings.Join(extraConfigs, "\n")) - if err != nil { - log.Fatal("InitProviderAndLoadCommonSettingsForTest: %v", err) - } - loadCommonSettingsFrom(CfgProvider) - if err := PrepareAppDataPath(); err != nil { - log.Fatal("Can not prepare APP_DATA_PATH: %v", err) - } - // register the dummy hash algorithm function used in the test fixtures - _ = hash.Register("dummy", hash.NewDummyHasher) - - PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy") -} - -// LoadCommonSettings loads common configurations from a configuration provider. -func LoadCommonSettings() { - loadCommonSettingsFrom(CfgProvider) -} - // loadCommonSettingsFrom loads common configurations from a configuration provider. func loadCommonSettingsFrom(cfg ConfigProvider) { // WARNNING: don't change the sequence except you know what you are doing. diff --git a/modules/setting/storage_test.go b/modules/setting/storage_test.go index 9c51bbc081c..5e213606e3f 100644 --- a/modules/setting/storage_test.go +++ b/modules/setting/storage_test.go @@ -19,7 +19,7 @@ MINIO_BUCKET = gitea-attachment STORAGE_TYPE = minio MINIO_ENDPOINT = my_minio:9000 ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -42,7 +42,7 @@ MINIO_BUCKET = gitea-attachment [storage.minio] MINIO_BUCKET = gitea ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -64,7 +64,7 @@ MINIO_BUCKET = gitea-minio [storage] MINIO_BUCKET = gitea ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -87,7 +87,7 @@ MINIO_BUCKET = gitea [storage] STORAGE_TYPE = local ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -99,7 +99,7 @@ STORAGE_TYPE = local } func Test_getStorageGetDefaults(t *testing.T) { - cfg, err := newConfigProviderFromData("") + cfg, err := NewConfigProviderFromData("") assert.NoError(t, err) sec := cfg.Section("attachment") @@ -120,7 +120,7 @@ MINIO_BUCKET = gitea-attachment [storage] MINIO_BUCKET = gitea-storage ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) { @@ -154,7 +154,7 @@ STORAGE_TYPE = lfs [storage.lfs] MINIO_BUCKET = gitea-storage ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) { @@ -178,7 +178,7 @@ func Test_getStorageInheritStorageType(t *testing.T) { [storage] STORAGE_TYPE = minio ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -193,7 +193,7 @@ func Test_getStorageInheritNameSectionType(t *testing.T) { [storage.attachments] STORAGE_TYPE = minio ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") diff --git a/modules/structs/repo.go b/modules/structs/repo.go index 9fcb0382624..43fd12a8bba 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -253,10 +253,16 @@ type CreateBranchRepoOption struct { // unique: true BranchName string `json:"new_branch_name" binding:"Required;GitRefName;MaxSize(100)"` + // Deprecated: true // Name of the old branch to create from // // unique: true OldBranchName string `json:"old_branch_name" binding:"GitRefName;MaxSize(100)"` + + // Name of the old branch/tag/commit to create from + // + // unique: true + OldRefName string `json:"old_ref_name" binding:"GitRefName;MaxSize(100)"` } // TransferRepoOption options when transfer a repository's ownership diff --git a/modules/structs/task.go b/modules/structs/task.go index c73e51b1ab0..ed11a33e28e 100644 --- a/modules/structs/task.go +++ b/modules/structs/task.go @@ -6,10 +6,7 @@ package structs // TaskType defines task type type TaskType int -// all kinds of task types -const ( - TaskTypeMigrateRepo TaskType = iota // migrate repository from external or local disk -) +const TaskTypeMigrateRepo TaskType = iota // migrate repository from external or local disk // Name returns the task type name func (taskType TaskType) Name() string { @@ -25,9 +22,9 @@ type TaskStatus int // enumerate all the kinds of task status const ( - TaskStatusQueue TaskStatus = iota // 0 task is queue + TaskStatusQueued TaskStatus = iota // 0 task is queued TaskStatusRunning // 1 task is running - TaskStatusStopped // 2 task is stopped + TaskStatusStopped // 2 task is stopped (never used) TaskStatusFailed // 3 task is failed TaskStatusFinished // 4 task is finished ) diff --git a/modules/templates/base.go b/modules/templates/base.go index 4254a569764..ef28cc03f45 100644 --- a/modules/templates/base.go +++ b/modules/templates/base.go @@ -5,43 +5,12 @@ package templates import ( "strings" - "time" "code.gitea.io/gitea/modules/assetfs" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) -// Vars represents variables to be render in golang templates -type Vars map[string]interface{} - -// Merge merges another vars to the current, another Vars will override the current -func (vars Vars) Merge(another map[string]interface{}) Vars { - for k, v := range another { - vars[k] = v - } - return vars -} - -// BaseVars returns all basic vars -func BaseVars() Vars { - startTime := time.Now() - return map[string]interface{}{ - "IsLandingPageHome": setting.LandingPageURL == setting.LandingPageHome, - "IsLandingPageExplore": setting.LandingPageURL == setting.LandingPageExplore, - "IsLandingPageOrganizations": setting.LandingPageURL == setting.LandingPageOrganizations, - - "ShowRegistrationButton": setting.Service.ShowRegistrationButton, - "ShowMilestonesDashboardPage": setting.Service.ShowMilestonesDashboardPage, - "ShowFooterVersion": setting.Other.ShowFooterVersion, - "DisableDownloadSourceArchives": setting.Repository.DisableDownloadSourceArchives, - - "EnableSwagger": setting.API.EnableSwagger, - "EnableOpenIDSignIn": setting.Service.EnableOpenIDSignIn, - "PageStartTime": startTime, - } -} - func AssetFS() *assetfs.LayeredFS { return assetfs.Layered(CustomAssets(), BuiltinAssets()) } diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 4abd94d46e5..7460f99feac 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -179,7 +179,6 @@ func NewFuncMap() template.FuncMap { // ----------------------------------------------------------------- // misc - "DiffLineTypeToStr": DiffLineTypeToStr, "ShortSha": base.ShortSha, "ActionContent2Commits": ActionContent2Commits, "IsMultilineCommitMessage": IsMultilineCommitMessage, diff --git a/modules/templates/util_misc.go b/modules/templates/util_misc.go index 599a0942ce7..d11251fcdf6 100644 --- a/modules/templates/util_misc.go +++ b/modules/templates/util_misc.go @@ -122,19 +122,6 @@ func ActionContent2Commits(act Actioner) *repository.PushCommits { return push } -// DiffLineTypeToStr returns diff line type name -func DiffLineTypeToStr(diffType int) string { - switch diffType { - case 2: - return "add" - case 3: - return "del" - case 4: - return "tag" - } - return "same" -} - // MigrationIcon returns a SVG name matching the service an issue/comment was migrated from func MigrationIcon(hostname string) string { switch hostname { diff --git a/modules/templates/util_render.go b/modules/templates/util_render.go index a59ddd3f175..a26c0531f81 100644 --- a/modules/templates/util_render.go +++ b/modules/templates/util_render.go @@ -20,6 +20,7 @@ import ( "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/markdown" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) // RenderCommitMessage renders commit message with XSS-safe and special links. @@ -133,7 +134,9 @@ func RenderLabel(ctx context.Context, label *issues_model.Label) template.HTML { labelScope := label.ExclusiveScope() textColor := "#111" - if label.UseLightTextColor() { + r, g, b := util.HexToRBGColor(label.Color) + // Determine if label text should be light or dark to be readable on background color + if util.UseLightTextOnBackground(r, g, b) { textColor = "#eee" } @@ -150,34 +153,30 @@ func RenderLabel(ctx context.Context, label *issues_model.Label) template.HTML { scopeText := RenderEmoji(ctx, labelScope) itemText := RenderEmoji(ctx, label.Name[len(labelScope)+1:]) - itemColor := label.Color - scopeColor := label.Color - if r, g, b, err := label.ColorRGB(); err == nil { - // Make scope and item background colors slightly darker and lighter respectively. - // More contrast needed with higher luminance, empirically tweaked. - luminance := (0.299*r + 0.587*g + 0.114*b) / 255 - contrast := 0.01 + luminance*0.03 - // Ensure we add the same amount of contrast also near 0 and 1. - darken := contrast + math.Max(luminance+contrast-1.0, 0.0) - lighten := contrast + math.Max(contrast-luminance, 0.0) - // Compute factor to keep RGB values proportional. - darkenFactor := math.Max(luminance-darken, 0.0) / math.Max(luminance, 1.0/255.0) - lightenFactor := math.Min(luminance+lighten, 1.0) / math.Max(luminance, 1.0/255.0) + // Make scope and item background colors slightly darker and lighter respectively. + // More contrast needed with higher luminance, empirically tweaked. + luminance := util.GetLuminance(r, g, b) + contrast := 0.01 + luminance*0.03 + // Ensure we add the same amount of contrast also near 0 and 1. + darken := contrast + math.Max(luminance+contrast-1.0, 0.0) + lighten := contrast + math.Max(contrast-luminance, 0.0) + // Compute factor to keep RGB values proportional. + darkenFactor := math.Max(luminance-darken, 0.0) / math.Max(luminance, 1.0/255.0) + lightenFactor := math.Min(luminance+lighten, 1.0) / math.Max(luminance, 1.0/255.0) - scopeBytes := []byte{ - uint8(math.Min(math.Round(r*darkenFactor), 255)), - uint8(math.Min(math.Round(g*darkenFactor), 255)), - uint8(math.Min(math.Round(b*darkenFactor), 255)), - } - itemBytes := []byte{ - uint8(math.Min(math.Round(r*lightenFactor), 255)), - uint8(math.Min(math.Round(g*lightenFactor), 255)), - uint8(math.Min(math.Round(b*lightenFactor), 255)), - } - - itemColor = "#" + hex.EncodeToString(itemBytes) - scopeColor = "#" + hex.EncodeToString(scopeBytes) + scopeBytes := []byte{ + uint8(math.Min(math.Round(r*darkenFactor), 255)), + uint8(math.Min(math.Round(g*darkenFactor), 255)), + uint8(math.Min(math.Round(b*darkenFactor), 255)), } + itemBytes := []byte{ + uint8(math.Min(math.Round(r*lightenFactor), 255)), + uint8(math.Min(math.Round(g*lightenFactor), 255)), + uint8(math.Min(math.Round(b*lightenFactor), 255)), + } + + itemColor := "#" + hex.EncodeToString(itemBytes) + scopeColor := "#" + hex.EncodeToString(scopeBytes) s := fmt.Sprintf(""+ "
%s
"+ diff --git a/modules/test/context_tests.go b/modules/test/context_tests.go index 4af76512505..5ba21261261 100644 --- a/modules/test/context_tests.go +++ b/modules/test/context_tests.go @@ -26,11 +26,12 @@ import ( ) // MockContext mock context for unit tests +// TODO: move this function to other packages, because it depends on "models" package func MockContext(t *testing.T, path string) *context.Context { resp := &mockResponseWriter{} ctx := context.Context{ Render: &mockRender{}, - Data: make(map[string]interface{}), + Data: make(middleware.ContextData), Flash: &middleware.Flash{ Values: make(url.Values), }, diff --git a/tests/testlogger.go b/modules/testlogger/testlogger.go similarity index 77% rename from tests/testlogger.go rename to modules/testlogger/testlogger.go index 7cac5bbe333..bf912f41dcf 100644 --- a/tests/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -1,7 +1,7 @@ // Copyright 2019 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package tests +package testlogger import ( "context" @@ -36,56 +36,64 @@ type testLoggerWriterCloser struct { t []*testing.TB } -func (w *testLoggerWriterCloser) setT(t *testing.TB) { +func (w *testLoggerWriterCloser) pushT(t *testing.TB) { w.Lock() w.t = append(w.t, t) w.Unlock() } func (w *testLoggerWriterCloser) Write(p []byte) (int, error) { + // There was a data race problem: the logger system could still try to output logs after the runner is finished. + // So we must ensure that the "t" in stack is still valid. w.RLock() + defer w.RUnlock() + var t *testing.TB if len(w.t) > 0 { t = w.t[len(w.t)-1] } - w.RUnlock() - if t != nil && *t != nil { - if len(p) > 0 && p[len(p)-1] == '\n' { - p = p[:len(p)-1] - } - defer func() { - err := recover() - if err == nil { - return - } - var errString string - errErr, ok := err.(error) - if ok { - errString = errErr.Error() - } else { - errString, ok = err.(string) - } - if !ok { - panic(err) - } - if !strings.HasPrefix(errString, "Log in goroutine after ") { - panic(err) - } - }() - - (*t).Log(string(p)) - return len(p), nil + if len(p) > 0 && p[len(p)-1] == '\n' { + p = p[:len(p)-1] } + + if t == nil || *t == nil { + return fmt.Fprintf(os.Stdout, "??? [Unknown Test] %s\n", p) + } + + defer func() { + err := recover() + if err == nil { + return + } + var errString string + errErr, ok := err.(error) + if ok { + errString = errErr.Error() + } else { + errString, ok = err.(string) + } + if !ok { + panic(err) + } + if !strings.HasPrefix(errString, "Log in goroutine after ") { + panic(err) + } + }() + + (*t).Log(string(p)) return len(p), nil } -func (w *testLoggerWriterCloser) Close() error { +func (w *testLoggerWriterCloser) popT() { w.Lock() if len(w.t) > 0 { w.t = w.t[:len(w.t)-1] } w.Unlock() +} + +func (w *testLoggerWriterCloser) Close() error { return nil } @@ -118,7 +126,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { } else { fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line) } - WriterCloser.setT(&t) + WriterCloser.pushT(&t) return func() { took := time.Since(start) if took > SlowTest { @@ -135,7 +143,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush) } }) - if err := queue.GetManager().FlushAll(context.Background(), 2*time.Minute); err != nil { + if err := queue.GetManager().FlushAll(context.Background(), time.Minute); err != nil { t.Errorf("Flushing queues failed with error %v", err) } timer.Stop() @@ -147,7 +155,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook) } } - _ = WriterCloser.Close() + WriterCloser.popT() } } @@ -195,7 +203,10 @@ func (log *TestLogger) GetName() string { } func init() { - log.Register("test", NewTestLogger) + const relFilePath = "modules/testlogger/testlogger.go" _, filename, _, _ := runtime.Caller(0) - prefix = strings.TrimSuffix(filename, "tests/integration/testlogger.go") + if !strings.HasSuffix(filename, relFilePath) { + panic("source code file path doesn't match expected: " + relFilePath) + } + prefix = strings.TrimSuffix(filename, relFilePath) } diff --git a/modules/timeutil/timestampnano.go b/modules/timeutil/timestampnano.go new file mode 100644 index 00000000000..4a9f7955b98 --- /dev/null +++ b/modules/timeutil/timestampnano.go @@ -0,0 +1,28 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package timeutil + +import ( + "time" + + "code.gitea.io/gitea/modules/setting" +) + +// TimeStampNano is for nano time in database, do not use it unless there is a real requirement. +type TimeStampNano int64 + +// TimeStampNanoNow returns now nano int64 +func TimeStampNanoNow() TimeStampNano { + return TimeStampNano(time.Now().UnixNano()) +} + +// AsTime convert timestamp as time.Time in Local locale +func (tsn TimeStampNano) AsTime() (tm time.Time) { + return tsn.AsTimeInLocation(setting.DefaultUILocation) +} + +// AsTimeInLocation convert timestamp as time.Time in Local locale +func (tsn TimeStampNano) AsTimeInLocation(loc *time.Location) time.Time { + return time.Unix(0, int64(tsn)).In(loc) +} diff --git a/modules/util/color.go b/modules/util/color.go new file mode 100644 index 00000000000..240b045c287 --- /dev/null +++ b/modules/util/color.go @@ -0,0 +1,65 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT +package util + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// Check similar implementation in web_src/js/utils/color.js and keep synchronization + +// Return R, G, B values defined in reletive luminance +func getLuminanceRGB(channel float64) float64 { + sRGB := channel / 255 + if sRGB <= 0.03928 { + return sRGB / 12.92 + } + return math.Pow((sRGB+0.055)/1.055, 2.4) +} + +// Get color as RGB values in 0..255 range from the hex color string (with or without #) +func HexToRBGColor(colorString string) (float64, float64, float64) { + hexString := colorString + if strings.HasPrefix(colorString, "#") { + hexString = colorString[1:] + } + // only support transfer of rgb, rgba, rrggbb and rrggbbaa + // if not in these formats, use default values 0, 0, 0 + if len(hexString) != 3 && len(hexString) != 4 && len(hexString) != 6 && len(hexString) != 8 { + return 0, 0, 0 + } + if len(hexString) == 3 || len(hexString) == 4 { + hexString = fmt.Sprintf("%c%c%c%c%c%c", hexString[0], hexString[0], hexString[1], hexString[1], hexString[2], hexString[2]) + } + if len(hexString) == 8 { + hexString = hexString[0:6] + } + color, err := strconv.ParseUint(hexString, 16, 64) + if err != nil { + return 0, 0, 0 + } + r := float64(uint8(0xFF & (uint32(color) >> 16))) + g := float64(uint8(0xFF & (uint32(color) >> 8))) + b := float64(uint8(0xFF & uint32(color))) + return r, g, b +} + +// return luminance given RGB channels +// Reference from: https://www.w3.org/WAI/GL/wiki/Relative_luminance +func GetLuminance(r, g, b float64) float64 { + R := getLuminanceRGB(r) + G := getLuminanceRGB(g) + B := getLuminanceRGB(b) + luminance := 0.2126*R + 0.7152*G + 0.0722*B + return luminance +} + +// Reference from: https://firsching.ch/github_labels.html +// In the future WCAG 3 APCA may be a better solution. +// Check if text should use light color based on RGB of background +func UseLightTextOnBackground(r, g, b float64) bool { + return GetLuminance(r, g, b) < 0.453 +} diff --git a/modules/util/color_test.go b/modules/util/color_test.go new file mode 100644 index 00000000000..d96ac36730a --- /dev/null +++ b/modules/util/color_test.go @@ -0,0 +1,65 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_HexToRBGColor(t *testing.T) { + cases := []struct { + colorString string + expectedR float64 + expectedG float64 + expectedB float64 + }{ + {"2b8685", 43, 134, 133}, + {"1e1", 17, 238, 17}, + {"#1e1", 17, 238, 17}, + {"1e16", 17, 238, 17}, + {"3bb6b3", 59, 182, 179}, + {"#3bb6b399", 59, 182, 179}, + {"#0", 0, 0, 0}, + {"#00000", 0, 0, 0}, + {"#1234567", 0, 0, 0}, + } + for n, c := range cases { + r, g, b := HexToRBGColor(c.colorString) + assert.Equal(t, c.expectedR, r, "case %d: error R should match: expected %f, but get %f", n, c.expectedR, r) + assert.Equal(t, c.expectedG, g, "case %d: error G should match: expected %f, but get %f", n, c.expectedG, g) + assert.Equal(t, c.expectedB, b, "case %d: error B should match: expected %f, but get %f", n, c.expectedB, b) + } +} + +func Test_UseLightTextOnBackground(t *testing.T) { + cases := []struct { + r float64 + g float64 + b float64 + expected bool + }{ + {215, 58, 74, true}, + {0, 117, 202, true}, + {207, 211, 215, false}, + {162, 238, 239, false}, + {112, 87, 255, true}, + {0, 134, 114, true}, + {228, 230, 105, false}, + {216, 118, 227, true}, + {255, 255, 255, false}, + {43, 134, 133, true}, + {43, 135, 134, true}, + {44, 135, 134, true}, + {59, 182, 179, true}, + {124, 114, 104, true}, + {126, 113, 108, true}, + {129, 112, 109, true}, + {128, 112, 112, true}, + } + for n, c := range cases { + result := UseLightTextOnBackground(c.r, c.g, c.b) + assert.Equal(t, c.expected, result, "case %d: error should match", n) + } +} diff --git a/modules/util/timer.go b/modules/util/timer.go index d598fde73ad..f9a7950709f 100644 --- a/modules/util/timer.go +++ b/modules/util/timer.go @@ -8,18 +8,6 @@ import ( "time" ) -// StopTimer is a utility function to safely stop a time.Timer and clean its channel -func StopTimer(t *time.Timer) bool { - stopped := t.Stop() - if !stopped { - select { - case <-t.C: - default: - } - } - return stopped -} - func Debounce(d time.Duration) func(f func()) { type debouncer struct { mu sync.Mutex diff --git a/modules/web/middleware/data.go b/modules/web/middleware/data.go index 43189940ee2..c1f0516d7d2 100644 --- a/modules/web/middleware/data.go +++ b/modules/web/middleware/data.go @@ -3,7 +3,63 @@ package middleware -// DataStore represents a data store -type DataStore interface { - GetData() map[string]interface{} +import ( + "context" + "time" + + "code.gitea.io/gitea/modules/setting" +) + +// ContextDataStore represents a data store +type ContextDataStore interface { + GetData() ContextData +} + +type ContextData map[string]any + +func (ds ContextData) GetData() map[string]any { + return ds +} + +func (ds ContextData) MergeFrom(other ContextData) ContextData { + for k, v := range other { + ds[k] = v + } + return ds +} + +const ContextDataKeySignedUser = "SignedUser" + +type contextDataKeyType struct{} + +var contextDataKey contextDataKeyType + +func WithContextData(c context.Context) context.Context { + return context.WithValue(c, contextDataKey, make(ContextData, 10)) +} + +func GetContextData(c context.Context) ContextData { + if ds, ok := c.Value(contextDataKey).(ContextData); ok { + return ds + } + return nil +} + +func CommonTemplateContextData() ContextData { + return ContextData{ + "IsLandingPageHome": setting.LandingPageURL == setting.LandingPageHome, + "IsLandingPageExplore": setting.LandingPageURL == setting.LandingPageExplore, + "IsLandingPageOrganizations": setting.LandingPageURL == setting.LandingPageOrganizations, + + "ShowRegistrationButton": setting.Service.ShowRegistrationButton, + "ShowMilestonesDashboardPage": setting.Service.ShowMilestonesDashboardPage, + "ShowFooterVersion": setting.Other.ShowFooterVersion, + "DisableDownloadSourceArchives": setting.Repository.DisableDownloadSourceArchives, + + "EnableSwagger": setting.API.EnableSwagger, + "EnableOpenIDSignIn": setting.Service.EnableOpenIDSignIn, + "PageStartTime": time.Now(), + + "RunModeIsProd": setting.IsProd, + } } diff --git a/modules/web/middleware/flash.go b/modules/web/middleware/flash.go index f2d7cc692d2..fa29ddeffc7 100644 --- a/modules/web/middleware/flash.go +++ b/modules/web/middleware/flash.go @@ -18,7 +18,7 @@ var FlashNow bool // Flash represents a one time data transfer between two requests. type Flash struct { - DataStore + DataStore ContextDataStore url.Values ErrorMsg, WarningMsg, InfoMsg, SuccessMsg string } @@ -34,7 +34,7 @@ func (f *Flash) set(name, msg string, current ...bool) { } if isShow { - f.GetData()["Flash"] = f + f.DataStore.GetData()["Flash"] = f } else { f.Set(name, msg) } diff --git a/modules/web/middleware/request.go b/modules/web/middleware/request.go index 34add27214b..0bb155df703 100644 --- a/modules/web/middleware/request.go +++ b/modules/web/middleware/request.go @@ -12,8 +12,3 @@ import ( func IsAPIPath(req *http.Request) bool { return strings.HasPrefix(req.URL.Path, "/api/") } - -// IsInternalPath returns true if the specified URL is an internal API path -func IsInternalPath(req *http.Request) bool { - return strings.HasPrefix(req.URL.Path, "/api/internal/") -} diff --git a/modules/web/route.go b/modules/web/route.go index 6fd60f4ca73..d801f1025c9 100644 --- a/modules/web/route.go +++ b/modules/web/route.go @@ -25,12 +25,12 @@ func Bind[T any](_ T) any { } // SetForm set the form object -func SetForm(data middleware.DataStore, obj interface{}) { +func SetForm(data middleware.ContextDataStore, obj interface{}) { data.GetData()["__form"] = obj } // GetForm returns the validate form information -func GetForm(data middleware.DataStore) interface{} { +func GetForm(data middleware.ContextDataStore) interface{} { return data.GetData()["__form"] } diff --git a/options/license/ASWF-Digital-Assets-1.0 b/options/license/ASWF-Digital-Assets-1.0 new file mode 100644 index 00000000000..27e45b19c99 --- /dev/null +++ b/options/license/ASWF-Digital-Assets-1.0 @@ -0,0 +1,17 @@ +ASWF Digital Assets License v1.0 + +License for (the "Asset Name"). + + Copyright . All rights reserved. + +Redistribution and use of these digital assets, with or without modification, solely for education, training, research, software and hardware development, performance benchmarking (including publication of benchmark results and permitting reproducibility of the benchmark results by third parties), or software and hardware product demonstrations, are permitted provided that the following conditions are met: + +1. Redistributions of these digital assets or any part of them must include the above copyright notice, this list of conditions and the disclaimer below. + +2. Publications showing images derived from these digital assets must include the above copyright notice. + +3. The names of copyright holder or the names of its contributors may NOT be used to promote or to imply endorsement, sponsorship, or affiliation with products developed or tested utilizing these digital assets or benchmarking results obtained from these digital assets, without prior written permission from copyright holder. + +4. The assets and their output may only be referred to as the Asset Name listed above, and your use of the Asset Name shall be solely to identify the digital assets. Other than as expressly permitted by this License, you may NOT use any trade names, trademarks, service marks, or product names of the copyright holder for any purpose. + +DISCLAIMER: THESE DIGITAL ASSETS ARE PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDER BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THESE DIGITAL ASSETS, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 0ca1e002e8e..5a922c0eda0 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -79,6 +79,7 @@ milestones = Milestones ok = OK cancel = Cancel +rerun = Re-run save = Save add = Add add_all = Add All @@ -308,6 +309,7 @@ repos = Repositories users = Users organizations = Organizations search = Search +go_to = Go to code = Code search.type.tooltip = Search type search.fuzzy = Fuzzy @@ -559,7 +561,7 @@ target_branch_not_exist = Target branch does not exist. [user] change_avatar = Change your avatar… -join_on = Joined on +joined_on = Joined on %s repositories = Repositories activity = Public Activity followers = Followers @@ -567,6 +569,7 @@ starred = Starred Repositories watched = Watched Repositories code = Code projects = Projects +overview = Overview following = Following follow = Follow unfollow = Unfollow @@ -754,8 +757,8 @@ ssh_principal_deletion_desc = Removing a SSH Certificate Principal revokes its a ssh_key_deletion_success = The SSH key has been removed. gpg_key_deletion_success = The GPG key has been removed. ssh_principal_deletion_success = The principal has been removed. -add_on = Added on -valid_until = Valid until +added_on = Added on %s +valid_until_date = Valid until %s valid_forever = Valid forever last_used = Last used on no_activity = No recent activity @@ -1038,7 +1041,7 @@ migrated_from_fake = Migrated From %[1]s migrate.migrate = Migrate From %s migrate.migrating = Migrating from %s ... migrate.migrating_failed = Migrating from %s failed. -migrate.migrating_failed.error = Error: %s +migrate.migrating_failed.error = Failed to migrate: %s migrate.migrating_failed_no_addr = Migration failed. migrate.github.description = Migrate data from github.com or other GitHub instances. migrate.git.description = Migrate a repository only from any Git service. @@ -1055,6 +1058,8 @@ migrate.migrating_labels = Migrating Labels migrate.migrating_releases = Migrating Releases migrate.migrating_issues = Migrating Issues migrate.migrating_pulls = Migrating Pull Requests +migrate.cancel_migrating_title = Cancel Migration +migrate.cancel_migrating_confirm = Do you want to cancel this migration? mirror_from = mirror of forked_from = forked from @@ -1184,7 +1189,7 @@ editor.filename_is_invalid = The filename is invalid: "%s". editor.branch_does_not_exist = Branch "%s" does not exist in this repository. editor.branch_already_exists = Branch "%s" already exists in this repository. editor.directory_is_a_file = Directory name "%s" is already used as a filename in this repository. -editor.file_is_a_symlink = "%s" is a symbolic link. Symbolic links cannot be edited in the web editor +editor.file_is_a_symlink = `"%s" is a symbolic link. Symbolic links cannot be edited in the web editor` editor.filename_is_a_directory = Filename "%s" is already used as a directory name in this repository. editor.file_editing_no_longer_exists = The file being edited, "%s", no longer exists in this repository. editor.file_deleting_no_longer_exists = The file being deleted, "%s", no longer exists in this repository. @@ -1907,6 +1912,7 @@ settings.sync_mirror = Synchronize Now settings.mirror_sync_in_progress = Mirror synchronization is in progress. Check back in a minute. settings.site = Website settings.update_settings = Update Settings +settings.update_mirror_settings = Update Mirror Settings settings.branches.switch_default_branch = Switch Default Branch settings.branches.update_default_branch = Update Default Branch settings.branches.add_new_rule = Add New Rule @@ -2006,6 +2012,7 @@ settings.delete_notices_2 = - This operation will permanently delete the %s? monitor.process.children = Children + monitor.queues = Queues monitor.queue = Queue: %s monitor.queue.name = Name @@ -3057,56 +3067,15 @@ monitor.queue.maxnumberworkers = Max Number of Workers monitor.queue.numberinqueue = Number in Queue monitor.queue.review = Review Config monitor.queue.review_add = Review/Add Workers -monitor.queue.configuration = Initial Configuration -monitor.queue.nopool.title = No Worker Pool -monitor.queue.nopool.desc = This queue wraps other queues and does not itself have a worker pool. -monitor.queue.wrapped.desc = A wrapped queue wraps a slow starting queue, buffering queued requests in a channel. It does not have a worker pool itself. -monitor.queue.persistable-channel.desc = A persistable-channel wraps two queues, a channel queue that has its own worker pool and a level queue for persisted requests from previous shutdowns. It does not have a worker pool itself. -monitor.queue.flush = Flush worker -monitor.queue.pool.timeout = Timeout -monitor.queue.pool.addworkers.title = Add Workers -monitor.queue.pool.addworkers.submit = Add Workers -monitor.queue.pool.addworkers.desc = Add Workers to this pool with or without a timeout. If you set a timeout these workers will be removed from the pool after the timeout has lapsed. -monitor.queue.pool.addworkers.numberworkers.placeholder = Number of Workers -monitor.queue.pool.addworkers.timeout.placeholder = Set to 0 for no timeout -monitor.queue.pool.addworkers.mustnumbergreaterzero = Number of Workers to add must be greater than zero -monitor.queue.pool.addworkers.musttimeoutduration = Timeout must be a golang duration eg. 5m or be 0 -monitor.queue.pool.flush.title = Flush Queue -monitor.queue.pool.flush.desc = Flush will add a worker that will terminate once the queue is empty, or it times out. -monitor.queue.pool.flush.submit = Add Flush Worker -monitor.queue.pool.flush.added = Flush Worker added for %[1]s -monitor.queue.pool.pause.title = Pause Queue -monitor.queue.pool.pause.desc = Pausing a Queue will prevent it from processing data -monitor.queue.pool.pause.submit = Pause Queue -monitor.queue.pool.resume.title = Resume Queue -monitor.queue.pool.resume.desc = Set this queue to resume work -monitor.queue.pool.resume.submit = Resume Queue - monitor.queue.settings.title = Pool Settings -monitor.queue.settings.desc = Pools dynamically grow with a boost in response to their worker queue blocking. These changes will not affect current worker groups. -monitor.queue.settings.timeout = Boost Timeout -monitor.queue.settings.timeout.placeholder = Currently %[1]v -monitor.queue.settings.timeout.error = Timeout must be a golang duration eg. 5m or be 0 -monitor.queue.settings.numberworkers = Boost Number of Workers -monitor.queue.settings.numberworkers.placeholder = Currently %[1]d -monitor.queue.settings.numberworkers.error = Number of Workers to add must be greater than or equal to zero +monitor.queue.settings.desc = Pools dynamically grow in response to their worker queue blocking. monitor.queue.settings.maxnumberworkers = Max Number of workers monitor.queue.settings.maxnumberworkers.placeholder = Currently %[1]d monitor.queue.settings.maxnumberworkers.error = Max number of workers must be a number monitor.queue.settings.submit = Update Settings monitor.queue.settings.changed = Settings Updated -monitor.queue.settings.blocktimeout = Current Block Timeout -monitor.queue.settings.blocktimeout.value = %[1]v - -monitor.queue.pool.none = This queue does not have a Pool -monitor.queue.pool.added = Worker Group Added -monitor.queue.pool.max_changed = Maximum number of workers changed -monitor.queue.pool.workers.title = Active Worker Groups -monitor.queue.pool.workers.none = No worker groups. -monitor.queue.pool.cancel = Shutdown Worker Group -monitor.queue.pool.cancelling = Worker Group shutting down -monitor.queue.pool.cancel_notices = Shutdown this group of %s workers? -monitor.queue.pool.cancel_desc = Leaving a queue without any worker groups may cause requests to block indefinitely. +monitor.queue.settings.remove_all_items = Remove all +monitor.queue.settings.remove_all_items_done = All items in the queue have been removed. notices.system_notice_list = System Notices notices.view_detail_header = View Notice Details @@ -3309,6 +3278,9 @@ pub.documentation = For more information on the Pub registry, see the documentation. +rpm.registry = Setup this registry from the command line: +rpm.install = To install the package, run the following command: +rpm.documentation = For more information on the RPM registry, see the documentation. rubygems.install = To install the package using gem, run the following command: rubygems.install2 = or add it to the Gemfile: rubygems.dependencies.runtime = Runtime Dependencies diff --git a/options/locale/locale_ru-RU.ini b/options/locale/locale_ru-RU.ini index 71eaaebcf13..e90c6e0059e 100644 --- a/options/locale/locale_ru-RU.ini +++ b/options/locale/locale_ru-RU.ini @@ -147,6 +147,7 @@ occurred=Произошла ошибка report_message=Если вы уверены, что это баг Gitea, пожалуйста, поищите задачу на GitHub или создайте новую при необходимости. missing_csrf=Некорректный запрос: отсутствует токен CSRF invalid_csrf=Некорректный запрос: неверный токен CSRF +not_found=Цель не найдена. network_error=Ошибка сети [startpage] @@ -421,7 +422,7 @@ issue_assigned.pull=@%[1]s назначил(а) вам запрос на сли issue_assigned.issue=@%[1]s назначил(а) вам задачу %[2]s в репозитории %[3]s. issue.x_mentioned_you=@%s упомянул(а) вас: -issue.action.force_push=%[1]s форсировал(а) отправку изменений %[2]s с %[3]s до %[4]s. +issue.action.force_push=%[1]s форсировал(а) отправку в %[2]s изменений %[4]s вместо %[3]s. issue.action.push_1=@%[1]s отправил(а) %[3]d изменение в %[2]s issue.action.push_n=@%[1]s отправил(а) %[3]d изменений в %[2]s issue.action.close=@%[1]s закрыл(а) #%[2]d. @@ -459,6 +460,7 @@ team_invite.text_3=Примечание: Это приглашение было [modal] yes=Да no=Нет +confirm=Подтвердить cancel=Отменить modify=Изменить @@ -914,6 +916,7 @@ readme_helper_desc=Это место, где вы можете написать auto_init=Инициализировать репозиторий (Добавляет .gitignore, LICENSE and README) trust_model_helper=Выберите модель доверия для проверки подписи. Возможные варианты: trust_model_helper_collaborator=Соавтор: Доверять подписям соавторов +trust_model_helper_committer=Автор коммита: доверять подписям, соответствующим авторам коммитов trust_model_helper_collaborator_committer=Соавтор+Коммитер: Доверять подписям соавторов, которые соответствуют автору коммита trust_model_helper_default=По умолчанию: используйте модель доверия по умолчанию для этой установки create_repo=Создать репозиторий @@ -1490,7 +1493,7 @@ issues.error_modifying_due_date=Не удалось изменить срок в issues.error_removing_due_date=Не удалось убрать срок выполнения. issues.push_commit_1=добавил(а) %d коммит %s issues.push_commits_n=добавил(а) %d коммитов %s -issues.force_push_codes=`принудительно залито %[1]s от %[2]s к %[4]s %[6]s` +issues.force_push_codes=`форсировал(а) отправку изменений %[1]s %[4]s вместо %[2]s %[6]s` issues.force_push_compare=Сравнить issues.due_date_form=гггг-мм-дд issues.due_date_form_add=Добавить срок выполнения @@ -2153,6 +2156,11 @@ settings.dismiss_stale_approvals=Отклонить устаревшие раз settings.dismiss_stale_approvals_desc=Когда новые коммиты, изменяющие содержимое запроса на слияние, отправляются в ветку, старые разрешения будут отклонены. settings.require_signed_commits=Требовать подписанные коммиты settings.require_signed_commits_desc=Отклонить отправку изменений в эту ветку, если они не подписаны или не проверяемы. +settings.protect_branch_name_pattern=Шаблон имени для защищённых веток +settings.protect_protected_file_patterns=Шаблоны защищённых файлов (разделённые точкой с запятой ';'): +settings.protect_protected_file_patterns_desc=Защищенные файлы нельзя изменить напрямую, даже если пользователь имеет право добавлять, редактировать или удалять файлы в этой ветке. Можно указать несколько шаблонов, разделяя их точкой с запятой (';'). О синтаксисе шаблонов читайте в документации github.com/gobwas/glob. Примеры: .drone.yml, /docs/**/*.txt. +settings.protect_unprotected_file_patterns=Шаблоны незащищённых файлов (разделённые точкой с запятой ';'): +settings.protect_unprotected_file_patterns_desc=Незащищенные файлы, которые допускается изменять напрямую, если пользователь имеет право на запись, несмотря на ограничение отправки изменений. Можно указать несколько шаблонов, разделяя их точкой с запятой (';'). О синтаксисе шаблонов читайте в документации github.com/gobwas/glob. Примеры: .drone.yml, /docs/**/*.txt. settings.add_protected_branch=Включить защиту settings.delete_protected_branch=Отключить защиту settings.update_protect_branch_success=Защита веток по правилу «%s» изменена. diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index f1921828349..d4360fe49a2 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -119,8 +119,26 @@ footer.software=关于软件 footer.links=链接 [heatmap] +number_of_contributions_in_the_last_12_months=一年内 %s 次贡献 +no_contributions=目前还没有贡献。 +less=更少的 +more=更多的 [editor] +buttons.heading.tooltip=添加标题 +buttons.bold.tooltip=添加粗体文本 +buttons.italic.tooltip=添加斜体文本 +buttons.quote.tooltip=引用文本 +buttons.code.tooltip=添加代码 +buttons.link.tooltip=添加链接 +buttons.list.unordered.tooltip=添加待办清单 +buttons.list.ordered.tooltip=添加编号列表 +buttons.list.task.tooltip=添加任务列表 +buttons.mention.tooltip=提及用户或团队 +buttons.ref.tooltip=引用一个问题或拉取请求 +buttons.switch_to_legacy.tooltip=使用旧版编辑器 +buttons.enable_monospace_font=启用等宽字体 +buttons.disable_monospace_font=禁用等宽字体 [filter] string.asc=A - Z @@ -234,6 +252,7 @@ install_btn_confirm=立即安装 test_git_failed=无法识别 'git' 命令：%v sqlite3_not_available=您所使用的发行版不支持 SQLite3，请从 %s 下载官方构建版，而不是 gobuild 版本。 invalid_db_setting=数据库设置无效: %v +invalid_db_table=数据库表 '%s' 无效： %v invalid_repo_path=仓库根目录设置无效：%v invalid_app_data_path=应用数据路径无效： %v run_user_not_match=运行用户名不是当前的用户名：%s -> %s @@ -299,6 +318,7 @@ repo_no_results=未找到匹配的仓库。 user_no_results=未找到匹配的用户。 org_no_results=未找到匹配的组织。 code_no_results=未找到与搜索字词匹配的源代码。 +code_search_results=“%s” 的搜索结果是 code_last_indexed_at=最后索引于 %s relevant_repositories_tooltip=派生的仓库，以及缺少主题、图标和描述的仓库将被隐藏。 relevant_repositories=只显示相关的仓库， 显示未过滤结果。 @@ -442,6 +462,7 @@ team_invite.text_3=注意：这是发送给 %[1]s 的邀请。如果您未曾收 [modal] yes=确认操作 no=取消操作 +confirm=确认 cancel=取消 modify=更新 @@ -476,6 +497,8 @@ size_error=长度必须为 %s。 min_size_error=长度最小为 %s 个字符。 max_size_error=长度最大为 %s 个字符。 email_error=不是一个有效的邮箱地址。 +url_error=`'%s' 不是一个有效的 URL。` +include_error=`必须包含子字符串 "%s"。` glob_pattern_error=`匹配模式无效：%s.` regex_pattern_error=`正则表达式无效：%s.` username_error=` 只能包含字母数字字符('0-9','a-z','A-Z'), 破折号 ('-'), 下划线 ('_') 和点 ('.'). 不能以非字母数字字符开头或结尾，并且不允许连续的非字母数字字符。` @@ -500,6 +523,7 @@ team_name_been_taken=团队名称已被使用。 team_no_units_error=至少选择一项仓库单元。 email_been_used=该电子邮件地址已在使用中。 email_invalid=此邮箱地址无效。 +openid_been_used=OpenID 地址 "%s" 已被使用。 username_password_incorrect=用户名或密码不正确。 password_complexity=密码未达到复杂程度要求: password_lowercase_one=至少一个小写字符 @@ -548,7 +572,12 @@ unfollow=取消关注 heatmap.loading=正在加载热图... user_bio=简历 disabled_public_activity=该用户已隐藏活动记录。 +email_visibility.limited=所有已认证用户均可看到您的电子邮件地址 +email_visibility.private=只有你本人和管理员可以看到你的电子邮件地址 +form.name_reserved=用户名 "%s" 被保留。 +form.name_pattern_not_allowed=用户名中不允许使用 "%s" 格式。 +form.name_chars_not_allowed=用户名 "%s" 包含无效字符。 [settings] profile=个人信息 @@ -579,6 +608,7 @@ location=所在地区 update_theme=更新主题 update_profile=更新信息 update_language=更新语言 +update_language_not_found=语言 %s 不可用。 update_language_success=语言已更新。 update_profile_success=您的资料信息已经更新 change_username=您的用户名已更改。 @@ -589,6 +619,9 @@ cancel=取消操作 language=界面语言 ui=主题 hidden_comment_types=隐藏的评论类型 +hidden_comment_types_description=此处选中的注释类型不会显示在问题页面中。比如，勾选”标签“删除所有 " 添加/删除的