You've already forked library-registry
mirror of
https://github.com/arduino/library-registry.git
synced 2025-07-05 21:21:14 +03:00
Move submission parser tool to dedicated repository
This commit is contained in:
45
.github/workflows/check-go.yml
vendored
45
.github/workflows/check-go.yml
vendored
@ -1,45 +0,0 @@
|
|||||||
name: Check Go code
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
paths:
|
|
||||||
- ".github/workflows/check-go.yml"
|
|
||||||
- "Taskfile.yml"
|
|
||||||
- "manager/**"
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- ".github/workflows/check-go.yml"
|
|
||||||
- "Taskfile.yml"
|
|
||||||
- "manager/**"
|
|
||||||
# See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#workflow_dispatch
|
|
||||||
workflow_dispatch:
|
|
||||||
# See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows#repository_dispatch
|
|
||||||
repository_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
check:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- name: Checkout local repository
|
|
||||||
uses: actions/checkout@v2
|
|
||||||
|
|
||||||
- name: Install Go
|
|
||||||
uses: actions/setup-go@v2
|
|
||||||
with:
|
|
||||||
go-version: "1.14"
|
|
||||||
|
|
||||||
- name: Install Taskfile
|
|
||||||
uses: arduino/actions/setup-taskfile@master
|
|
||||||
with:
|
|
||||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
|
||||||
version: 3.x
|
|
||||||
|
|
||||||
- name: Lint
|
|
||||||
run: task go:lint
|
|
||||||
|
|
||||||
- name: Run tests
|
|
||||||
run: task go:test
|
|
||||||
|
|
||||||
- name: Check formatting
|
|
||||||
run: task go:check-formatting
|
|
29
.github/workflows/manage-prs.yml
vendored
29
.github/workflows/manage-prs.yml
vendored
@ -1,5 +1,8 @@
|
|||||||
name: Manage PRs
|
name: Manage PRs
|
||||||
|
|
||||||
|
env:
|
||||||
|
SUBMISSION_PARSER_VERSION: 1.0.0-rc2 # See: https://github.com/arduino/library-manager-submission-parser/releases
|
||||||
|
|
||||||
on:
|
on:
|
||||||
# pull_request_target trigger is used instead of pull_request so the token will have the write permissions needed to comment and merge.
|
# pull_request_target trigger is used instead of pull_request so the token will have the write permissions needed to comment and merge.
|
||||||
# Note that this means the version of the workflow from the PR base ref will be used as opposed to the head ref, as is the case with pull_request triggered workflows.
|
# Note that this means the version of the workflow from the PR base ref will be used as opposed to the head ref, as is the case with pull_request triggered workflows.
|
||||||
@ -25,7 +28,6 @@ jobs:
|
|||||||
contains(github.event.comment.body, 'ArduinoBot')
|
contains(github.event.comment.body, 'ArduinoBot')
|
||||||
)
|
)
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Dummy step to make job valid
|
- name: Dummy step to make job valid
|
||||||
run: ""
|
run: ""
|
||||||
@ -77,27 +79,15 @@ jobs:
|
|||||||
index-entry: ${{ steps.parse-request.outputs.index-entry }}
|
index-entry: ${{ steps.parse-request.outputs.index-entry }}
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Set environment variables
|
|
||||||
run: |
|
|
||||||
# See: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
|
|
||||||
echo "MANAGER_PATH=${{ runner.temp }}/manager" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Checkout local repository
|
- name: Checkout local repository
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
|
||||||
- name: Install Go
|
- name: Download submission parser
|
||||||
uses: actions/setup-go@v2
|
id: download-parser
|
||||||
|
uses: carlosperate/download-file-action@v1.0.3
|
||||||
- name: Install Taskfile
|
|
||||||
uses: arduino/actions/setup-taskfile@master
|
|
||||||
with:
|
with:
|
||||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
file-url: https://github.com/arduino/library-registry-submission-parser/releases/download/${{ env.SUBMISSION_PARSER_VERSION }}/parser
|
||||||
version: 3.x
|
location: ${{ runner.temp }}
|
||||||
|
|
||||||
- name: Build manager
|
|
||||||
env:
|
|
||||||
GO_BUILD_FLAGS: -o $MANAGER_PATH
|
|
||||||
run: task go:build
|
|
||||||
|
|
||||||
- name: Download diff
|
- name: Download diff
|
||||||
uses: actions/download-artifact@v2
|
uses: actions/download-artifact@v2
|
||||||
@ -108,7 +98,8 @@ jobs:
|
|||||||
- name: Parse request
|
- name: Parse request
|
||||||
id: parse-request
|
id: parse-request
|
||||||
run: |
|
run: |
|
||||||
REQUEST="$("$MANAGER_PATH" --diffpath="${{ needs.diff.outputs.path }}/${{ needs.diff.outputs.filename }}" --repopath="${{ github.workspace }}" --listname="repositories.txt")"
|
chmod u+x "${{ steps.download-parser.outputs.file-path }}"
|
||||||
|
REQUEST="$("${{ steps.download-parser.outputs.file-path }}" --diffpath="${{ needs.diff.outputs.path }}/${{ needs.diff.outputs.filename }}" --repopath="${{ github.workspace }}" --listname="repositories.txt")"
|
||||||
# Due to limitations of the GitHub Actions workflow system, dedicated outputs must be created for use in certain workflow fields.
|
# Due to limitations of the GitHub Actions workflow system, dedicated outputs must be created for use in certain workflow fields.
|
||||||
echo "::set-output name=type::$(echo "$REQUEST" | jq -r -c '.type')"
|
echo "::set-output name=type::$(echo "$REQUEST" | jq -r -c '.type')"
|
||||||
echo "::set-output name=submissions::$(echo "$REQUEST" | jq -c '.submissions')"
|
echo "::set-output name=submissions::$(echo "$REQUEST" | jq -c '.submissions')"
|
||||||
|
13
.gitignore
vendored
13
.gitignore
vendored
@ -1,13 +0,0 @@
|
|||||||
# Build artifacts
|
|
||||||
/manager/manager
|
|
||||||
/manager/manager.exe
|
|
||||||
|
|
||||||
# Test artifacts
|
|
||||||
coverage_unit.txt
|
|
||||||
|
|
||||||
# IDEs
|
|
||||||
.idea/
|
|
||||||
.vscode/
|
|
||||||
*.bak
|
|
||||||
*.code-workspace
|
|
||||||
*.sublime-workspace
|
|
62
Taskfile.yml
62
Taskfile.yml
@ -1,59 +1,9 @@
|
|||||||
version: "3"
|
version: "3"
|
||||||
|
|
||||||
vars:
|
vars:
|
||||||
DEFAULT_GO_PACKAGES:
|
|
||||||
sh: echo `cd manager && go list ./... | tr '\n' ' '`
|
|
||||||
DEFAULT_GO_PATHS:
|
|
||||||
sh: echo '`cd manager && go list -f '{{"{{"}}.Dir{{"}}"}}' ./...`'
|
|
||||||
|
|
||||||
PRETTIER: prettier@2.1.2
|
PRETTIER: prettier@2.1.2
|
||||||
|
|
||||||
tasks:
|
tasks:
|
||||||
go:build:
|
|
||||||
desc: Build the project
|
|
||||||
dir: manager
|
|
||||||
cmds:
|
|
||||||
- go build -v {{.GO_BUILD_FLAGS}}
|
|
||||||
|
|
||||||
check:
|
|
||||||
desc: Test, lint, and check formatting of everything
|
|
||||||
deps:
|
|
||||||
- task: go:check
|
|
||||||
- task: docs:check
|
|
||||||
- task: config:check
|
|
||||||
|
|
||||||
go:check:
|
|
||||||
desc: Test, lint, and check formatting of Go code
|
|
||||||
deps:
|
|
||||||
- task: go:lint
|
|
||||||
- task: go:test
|
|
||||||
- task: go:check-formatting
|
|
||||||
|
|
||||||
go:lint:
|
|
||||||
desc: Lint Go code
|
|
||||||
dir: manager
|
|
||||||
cmds:
|
|
||||||
- go vet {{default .DEFAULT_GO_PACKAGES .GO_PACKAGES}}
|
|
||||||
- go get golang.org/x/lint/golint
|
|
||||||
- |
|
|
||||||
GOLINT_PATH="$(go list -f '{{"{{"}}.Target{{"}}"}}' golang.org/x/lint/golint || echo "false")"
|
|
||||||
"$GOLINT_PATH" {{default "-min_confidence 0.8 -set_exit_status" .GO_LINT_FLAGS}} "{{default .DEFAULT_GO_PACKAGES .GO_PACKAGES}}"
|
|
||||||
|
|
||||||
go:test:
|
|
||||||
desc: Run unit tests
|
|
||||||
dir: manager
|
|
||||||
cmds:
|
|
||||||
- go test -v -short -run '{{default ".*" .GO_TEST_REGEX}}' {{default "-timeout 10m -coverpkg=./... -covermode=atomic" .GO_TEST_FLAGS}} -coverprofile=coverage_unit.txt {{default .DEFAULT_GO_PACKAGES .GO_PACKAGES}}
|
|
||||||
|
|
||||||
go:check-formatting:
|
|
||||||
desc: Check Go code formatting
|
|
||||||
dir: manager
|
|
||||||
cmds:
|
|
||||||
- |
|
|
||||||
RESULTS="$(gofmt -l {{default .DEFAULT_GO_PATHS .GO_PATHS}})"
|
|
||||||
echo "$RESULTS"
|
|
||||||
test -z "$RESULTS"
|
|
||||||
|
|
||||||
docs:check:
|
docs:check:
|
||||||
desc: Lint and check formatting of documentation files
|
desc: Lint and check formatting of documentation files
|
||||||
deps:
|
deps:
|
||||||
@ -119,18 +69,6 @@ tasks:
|
|||||||
cmds:
|
cmds:
|
||||||
- npx {{.PRETTIER}} --check "**/*.{yml,yaml,json}"
|
- npx {{.PRETTIER}} --check "**/*.{yml,yaml,json}"
|
||||||
|
|
||||||
format:
|
|
||||||
desc: Format all files
|
|
||||||
deps:
|
|
||||||
- task: go:format
|
|
||||||
- task: docs:format
|
|
||||||
- task: config:format
|
|
||||||
|
|
||||||
go:format:
|
|
||||||
desc: Format Go code
|
|
||||||
cmds:
|
|
||||||
- gofmt -l -w {{default .DEFAULT_GO_PATHS .GO_PATHS}}
|
|
||||||
|
|
||||||
docs:format:
|
docs:format:
|
||||||
desc: Format documentation files
|
desc: Format documentation files
|
||||||
cmds:
|
cmds:
|
||||||
|
@ -1,10 +0,0 @@
|
|||||||
module github.com/arduino/library-manager-list/manager
|
|
||||||
|
|
||||||
go 1.14
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/arduino/go-paths-helper v1.5.0
|
|
||||||
github.com/arduino/go-properties-orderedmap v1.4.0
|
|
||||||
github.com/sourcegraph/go-diff v0.6.1
|
|
||||||
github.com/stretchr/testify v1.3.0
|
|
||||||
)
|
|
@ -1,22 +0,0 @@
|
|||||||
github.com/arduino/go-paths-helper v1.0.1/go.mod h1:HpxtKph+g238EJHq4geEPv9p+gl3v5YYu35Yb+w31Ck=
|
|
||||||
github.com/arduino/go-paths-helper v1.5.0 h1:RVo189hD+GhUS1rQ3gixwK1nSbvVR8MGIGa7Gxv2bdM=
|
|
||||||
github.com/arduino/go-paths-helper v1.5.0/go.mod h1:V82BWgAAp4IbmlybxQdk9Bpkz8M4Qyx+RAFKaG9NuvU=
|
|
||||||
github.com/arduino/go-properties-orderedmap v1.4.0 h1:YEbbzPqm1gXWDM/Jaq8tlvmh09z2qeHPJTUw9/VA4Dk=
|
|
||||||
github.com/arduino/go-properties-orderedmap v1.4.0/go.mod h1:DKjD2VXY/NZmlingh4lSFMEYCVubfeArCsGPGDwb2yk=
|
|
||||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
||||||
github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
|
|
||||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
||||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
|
||||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
|
||||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
|
||||||
github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ=
|
|
||||||
github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
|
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
|
||||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
|
||||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
386
manager/main.go
386
manager/main.go
@ -1,386 +0,0 @@
|
|||||||
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
|
|
||||||
//
|
|
||||||
// This software is released under the GNU General Public License version 3.
|
|
||||||
// The terms of this license can be found at:
|
|
||||||
// https://www.gnu.org/licenses/gpl-3.0.en.html
|
|
||||||
//
|
|
||||||
// You can be released from the requirements of the above licenses by purchasing
|
|
||||||
// a commercial license. Buying such a license is mandatory if you want to
|
|
||||||
// modify or otherwise use the software for commercial activities involving the
|
|
||||||
// Arduino software without disclosing the source code of your own applications.
|
|
||||||
// To purchase a commercial license, send an email to license@arduino.cc.
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/sourcegraph/go-diff/diff"
|
|
||||||
|
|
||||||
"github.com/arduino/go-paths-helper"
|
|
||||||
properties "github.com/arduino/go-properties-orderedmap"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Git hosts that are supported for library repositories.
|
|
||||||
var supportedHosts []string = []string{
|
|
||||||
"bitbucket.org",
|
|
||||||
"github.com",
|
|
||||||
"gitlab.com",
|
|
||||||
}
|
|
||||||
|
|
||||||
// Libraries under these organizations will have the "Arduino" type and be linted with Arduino Lint in the "official" setting.
|
|
||||||
var officialOrganizations []string = []string{
|
|
||||||
"github.com/arduino",
|
|
||||||
"github.com/arduino-libraries",
|
|
||||||
"github.com/bcmi-labs",
|
|
||||||
"github.com/vidor-libraries",
|
|
||||||
}
|
|
||||||
|
|
||||||
// Libraries under these organizations will have the "Partner" type.
|
|
||||||
var partnerOrganizations []string = []string{
|
|
||||||
"github.com/Azure",
|
|
||||||
"github.com/ms-iot",
|
|
||||||
"github.com/ameltech",
|
|
||||||
}
|
|
||||||
|
|
||||||
// Libraries under these organizations will have the "Recommended" type.
|
|
||||||
var recommendedOrganizations []string = []string{
|
|
||||||
"github.com/adafruit",
|
|
||||||
}
|
|
||||||
|
|
||||||
// requestType is the type of the request data.
|
|
||||||
type requestType struct {
|
|
||||||
Type string `json:"type"` // Request type.
|
|
||||||
Submissions []submissionType `json:"submissions"` // Data for submitted libraries.
|
|
||||||
IndexEntry string `json:"indexEntry"` // Entry that will be made to the Library Manager index source file when the submission is accepted.
|
|
||||||
}
|
|
||||||
|
|
||||||
// submissionType is the type of the data for each individual library submitted in the request.
|
|
||||||
type submissionType struct {
|
|
||||||
SubmissionURL string `json:"submissionURL"` // Library repository URL as submitted by user. Used to identify the submission to the user.
|
|
||||||
NormalizedURL string `json:"normalizedURL"` // Submission URL in the standardized format that will be used in the index entry.
|
|
||||||
Name string `json:"name"` // Library name.
|
|
||||||
Official bool `json:"official"` // Whether the library is official.
|
|
||||||
Tag string `json:"tag"` // Name of the submission repository's latest tag, which is used as the basis for the index entry and validation.
|
|
||||||
Error string `json:"error"` // Error message.
|
|
||||||
}
|
|
||||||
|
|
||||||
// Command line flags.
|
|
||||||
var diffPathArgument = flag.String("diffpath", "", "")
|
|
||||||
var repoPathArgument = flag.String("repopath", "", "")
|
|
||||||
var listNameArgument = flag.String("listname", "", "")
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
// Validate flag input.
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if *diffPathArgument == "" {
|
|
||||||
errorExit("--diffpath flag is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
if *repoPathArgument == "" {
|
|
||||||
errorExit("--repopath flag is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
if *listNameArgument == "" {
|
|
||||||
errorExit("--listname flag is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
diffPath := paths.New(*diffPathArgument)
|
|
||||||
exist, err := diffPath.ExistCheck()
|
|
||||||
if !exist {
|
|
||||||
errorExit("diff file not found")
|
|
||||||
}
|
|
||||||
|
|
||||||
listPath := paths.New(*repoPathArgument, *listNameArgument)
|
|
||||||
exist, err = listPath.ExistCheck()
|
|
||||||
if !exist {
|
|
||||||
errorExit(fmt.Sprintf("list file %s not found", listPath))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse the PR diff.
|
|
||||||
rawDiff, err := diffPath.ReadFile()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
var request requestType
|
|
||||||
var submissionURLs []string
|
|
||||||
request.Type, submissionURLs = parseDiff(rawDiff, *listNameArgument)
|
|
||||||
|
|
||||||
// Process the submissions.
|
|
||||||
var indexEntries []string
|
|
||||||
for _, submissionURL := range submissionURLs {
|
|
||||||
submission, indexEntry := populateSubmission(submissionURL, listPath)
|
|
||||||
request.Submissions = append(request.Submissions, submission)
|
|
||||||
indexEntries = append(indexEntries, indexEntry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Assemble the index entry for the submissions.
|
|
||||||
request.IndexEntry = strings.Join(indexEntries, "%0A")
|
|
||||||
|
|
||||||
// Marshal the request data into a JSON document.
|
|
||||||
var marshalledRequest bytes.Buffer
|
|
||||||
jsonEncoder := json.NewEncoder(io.Writer(&marshalledRequest))
|
|
||||||
// By default, the json package HTML-sanitizes strings during marshalling (https://golang.org/pkg/encoding/json/#Marshal)
|
|
||||||
// It's not possible to change this behavior when using the simple json.MarshalIndent() approach.
|
|
||||||
jsonEncoder.SetEscapeHTML(false)
|
|
||||||
jsonEncoder.SetIndent("", "") // Single line.
|
|
||||||
err = jsonEncoder.Encode(request)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println(marshalledRequest.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// errorExit prints the error message in a standardized format and exits with status 1.
|
|
||||||
func errorExit(message string) {
|
|
||||||
fmt.Printf("ERROR: %s\n", message)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseDiff parses the request diff and returns the request type and list of submission URLs.
|
|
||||||
func parseDiff(rawDiff []byte, listName string) (string, []string) {
|
|
||||||
var submissionURLs []string
|
|
||||||
|
|
||||||
diffs, err := diff.ParseMultiFileDiff(rawDiff)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (len(diffs) != 1) || (diffs[0].OrigName[2:] != listName) || (diffs[0].OrigName[2:] != diffs[0].NewName[2:]) { // Git diffs have a a/ or b/ prefix on file names.
|
|
||||||
// This is not a Library Manager submission.
|
|
||||||
return "other", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var addedCount int
|
|
||||||
var deletedCount int
|
|
||||||
// Get the added URLs from the diff
|
|
||||||
for _, hunk := range diffs[0].Hunks {
|
|
||||||
hunkBody := string(hunk.Body)
|
|
||||||
for _, rawDiffLine := range strings.Split(hunkBody, "\n") {
|
|
||||||
diffLine := strings.TrimRight(rawDiffLine, " \t")
|
|
||||||
if len(diffLine) < 2 {
|
|
||||||
continue // Ignore blank lines.
|
|
||||||
}
|
|
||||||
|
|
||||||
switch diffLine[0] {
|
|
||||||
case '+':
|
|
||||||
addedCount++
|
|
||||||
submissionURLs = append(submissionURLs, strings.TrimSpace(diffLine[1:]))
|
|
||||||
case '-':
|
|
||||||
deletedCount++
|
|
||||||
default:
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var requestType string
|
|
||||||
if addedCount > 0 && deletedCount == 0 {
|
|
||||||
requestType = "submission"
|
|
||||||
} else if addedCount == 0 && deletedCount > 0 {
|
|
||||||
requestType = "removal"
|
|
||||||
} else {
|
|
||||||
requestType = "modification"
|
|
||||||
}
|
|
||||||
|
|
||||||
return requestType, submissionURLs
|
|
||||||
}
|
|
||||||
|
|
||||||
// populateSubmission does the checks on the submission that aren't provided by Arduino Lint and gathers the necessary data on it.
|
|
||||||
func populateSubmission(submissionURL string, listPath *paths.Path) (submissionType, string) {
|
|
||||||
indexSourceSeparator := "|"
|
|
||||||
var submission submissionType
|
|
||||||
|
|
||||||
submission.SubmissionURL = submissionURL
|
|
||||||
|
|
||||||
// Normalize and validate submission URL.
|
|
||||||
submissionURLObject, err := url.Parse(submission.SubmissionURL)
|
|
||||||
if err != nil {
|
|
||||||
submission.Error = fmt.Sprintf("Invalid submission URL (%s)", err)
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if URL is accessible.
|
|
||||||
httpResponse, err := http.Get(submissionURLObject.String())
|
|
||||||
if err != nil {
|
|
||||||
submission.Error = fmt.Sprintf("Unable to load submission URL: %s", err)
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
if httpResponse.StatusCode != http.StatusOK {
|
|
||||||
submission.Error = "Unable to load submission URL. Is the repository public?"
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resolve redirects and normalize.
|
|
||||||
normalizedURLObject := normalizeURL(httpResponse.Request.URL)
|
|
||||||
|
|
||||||
submission.NormalizedURL = normalizedURLObject.String()
|
|
||||||
|
|
||||||
// Check if URL is from a supported Git host.
|
|
||||||
if !uRLIsUnder(normalizedURLObject, supportedHosts) {
|
|
||||||
submission.Error = normalizedURLObject.Host + " is not currently supported as a Git hosting website for Library Manager.%0ASee: https://github.com/arduino/Arduino/wiki/Library-Manager-FAQ#how-can-i-add-my-library-to-library-manager"
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if URL is a Git repository
|
|
||||||
err = exec.Command("git", "ls-remote", normalizedURLObject.String()).Run()
|
|
||||||
if err != nil {
|
|
||||||
if _, ok := err.(*exec.ExitError); ok {
|
|
||||||
submission.Error = "Submission URL is not a Git clone URL (e.g., https://github.com/arduino-libraries/Servo)."
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the URL is already in the index.
|
|
||||||
listLines, err := listPath.ReadFileAsLines()
|
|
||||||
occurrences := 0
|
|
||||||
for _, listURL := range listLines {
|
|
||||||
listURLObject, err := url.Parse(strings.TrimSpace(listURL))
|
|
||||||
if err != nil {
|
|
||||||
panic(err) // All list items have already passed parsing so something is broken if this happens.
|
|
||||||
}
|
|
||||||
|
|
||||||
normalizedListURLObject := normalizeURL(listURLObject)
|
|
||||||
if normalizedListURLObject.String() == normalizedURLObject.String() {
|
|
||||||
occurrences++
|
|
||||||
if occurrences > 1 {
|
|
||||||
submission.Error = "Submission URL is already in the Library Manager index."
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine the library types attributes.
|
|
||||||
submission.Official = uRLIsUnder(normalizedURLObject, officialOrganizations)
|
|
||||||
var types []string
|
|
||||||
if submission.Official {
|
|
||||||
types = append(types, "Arduino")
|
|
||||||
}
|
|
||||||
if uRLIsUnder(normalizedURLObject, partnerOrganizations) {
|
|
||||||
types = append(types, "Partner")
|
|
||||||
}
|
|
||||||
if uRLIsUnder(normalizedURLObject, recommendedOrganizations) {
|
|
||||||
types = append(types, "Recommended")
|
|
||||||
}
|
|
||||||
if types == nil {
|
|
||||||
types = append(types, "Contributed")
|
|
||||||
}
|
|
||||||
|
|
||||||
submissionClonePath, err := paths.MkTempDir("", "")
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = exec.Command("git", "clone", "--depth", "1", normalizedURLObject.String(), submissionClonePath.String()).Run()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine latest tag name in submission repo
|
|
||||||
err = os.Chdir(submissionClonePath.String())
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
err = exec.Command("git", "fetch", "--tags").Run()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
tagList, err := exec.Command("git", "rev-list", "--tags", "--max-count=1").Output()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
if string(tagList) == "" {
|
|
||||||
submission.Error = "The repository has no tags. You need to create a [release](https://docs.github.com/en/github/administering-a-repository/managing-releases-in-a-repository) or [tag](https://git-scm.com/docs/git-tag) that matches the `version` value in the library's library.properties file."
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
latestTag, err := exec.Command("git", "describe", "--tags", strings.TrimSpace(string(tagList))).Output()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
submission.Tag = strings.TrimSpace(string(latestTag))
|
|
||||||
|
|
||||||
// Checkout latest tag.
|
|
||||||
err = exec.Command("git", "checkout", submission.Tag).Run()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get submission library name. It is necessary to record this in the index source entry because the library is locked to this name.
|
|
||||||
libraryPropertiesPath := submissionClonePath.Join("library.properties")
|
|
||||||
if !libraryPropertiesPath.Exist() {
|
|
||||||
submission.Error = "Library is missing a library.properties metadata file."
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
libraryProperties, err := properties.LoadFromPath(libraryPropertiesPath)
|
|
||||||
if err != nil {
|
|
||||||
submission.Error = fmt.Sprintf("Invalid library.properties file (%s)", err)
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
var ok bool
|
|
||||||
submission.Name, ok = libraryProperties.GetOk("name")
|
|
||||||
if !ok {
|
|
||||||
submission.Error = "library.properties is missing a name field"
|
|
||||||
return submission, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// Assemble Library Manager index source entry string
|
|
||||||
indexEntry := strings.Join(
|
|
||||||
[]string{
|
|
||||||
submission.NormalizedURL,
|
|
||||||
strings.Join(types, ","),
|
|
||||||
submission.Name,
|
|
||||||
},
|
|
||||||
indexSourceSeparator,
|
|
||||||
)
|
|
||||||
|
|
||||||
return submission, indexEntry
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalizeURL converts the URL into the standardized format used in the index.
|
|
||||||
func normalizeURL(rawURL *url.URL) url.URL {
|
|
||||||
normalizedPath := strings.TrimRight(rawURL.Path, "/")
|
|
||||||
if !strings.HasSuffix(normalizedPath, ".git") {
|
|
||||||
normalizedPath += ".git"
|
|
||||||
}
|
|
||||||
|
|
||||||
return url.URL{
|
|
||||||
Scheme: "https",
|
|
||||||
Host: rawURL.Host,
|
|
||||||
Path: normalizedPath,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func uRLIsUnder(childURL url.URL, parentCandidates []string) bool {
|
|
||||||
for _, parentCandidate := range parentCandidates {
|
|
||||||
if !strings.HasSuffix(parentCandidate, "/") {
|
|
||||||
parentCandidate += "/"
|
|
||||||
}
|
|
||||||
parentCandidateURL, err := url.Parse("https://" + parentCandidate)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
isUnderPath, err := paths.New(childURL.Path).IsInsideDir(paths.New(parentCandidateURL.Path))
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
if (childURL.Host == parentCandidateURL.Host) && isUnderPath {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
@ -1,194 +0,0 @@
|
|||||||
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
|
|
||||||
//
|
|
||||||
// This software is released under the GNU General Public License version 3.
|
|
||||||
// The terms of this license can be found at:
|
|
||||||
// https://www.gnu.org/licenses/gpl-3.0.en.html
|
|
||||||
//
|
|
||||||
// You can be released from the requirements of the above licenses by purchasing
|
|
||||||
// a commercial license. Buying such a license is mandatory if you want to
|
|
||||||
// modify or otherwise use the software for commercial activities involving the
|
|
||||||
// Arduino software without disclosing the source code of your own applications.
|
|
||||||
// To purchase a commercial license, send an email to license@arduino.cc.
|
|
||||||
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/url"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Test_parseDiff(t *testing.T) {
|
|
||||||
testName := "Multiple files"
|
|
||||||
diff := []byte(`
|
|
||||||
diff --git a/README.md b/README.md
|
|
||||||
index d4edde0..807b76d 100644
|
|
||||||
--- a/README.md
|
|
||||||
+++ b/README.md
|
|
||||||
@@ -1,0 +2 @@
|
|
||||||
+hello
|
|
||||||
diff --git a/repositories.txt b/repositories.txt
|
|
||||||
index cff484d..e14c179 100644
|
|
||||||
--- a/repositories.txt
|
|
||||||
+++ b/repositories.txt
|
|
||||||
@@ -8,0 +9 @@ https://github.com/arduino-libraries/Ethernet
|
|
||||||
+https://github.com/foo/bar
|
|
||||||
`)
|
|
||||||
|
|
||||||
requestType, submissionURLs := parseDiff(diff, "repositories.txt")
|
|
||||||
assert.Equal(t, "other", requestType, testName)
|
|
||||||
assert.Nil(t, submissionURLs, testName)
|
|
||||||
|
|
||||||
testName = "Not list"
|
|
||||||
diff = []byte(`
|
|
||||||
diff --git a/README.md b/README.md
|
|
||||||
index d4edde0..807b76d 100644
|
|
||||||
--- a/README.md
|
|
||||||
+++ b/README.md
|
|
||||||
@@ -1 +1,2 @@
|
|
||||||
# Arduino Library Manager list
|
|
||||||
+hello
|
|
||||||
`)
|
|
||||||
|
|
||||||
requestType, submissionURLs = parseDiff(diff, "repositories.txt")
|
|
||||||
assert.Equal(t, "other", requestType, testName)
|
|
||||||
assert.Nil(t, submissionURLs, testName)
|
|
||||||
|
|
||||||
testName = "List filename change"
|
|
||||||
diff = []byte(`
|
|
||||||
diff --git a/repositories.txt b/foobar.txt
|
|
||||||
similarity index 99%
|
|
||||||
rename from repositories.txt
|
|
||||||
rename to foobar.txt
|
|
||||||
index cff484d..e14c179 100644
|
|
||||||
--- a/repositories.txt
|
|
||||||
+++ b/foobar.txt
|
|
||||||
@@ -8,0 +9 @@ https://github.com/arduino-libraries/Ethernet
|
|
||||||
+https://github.com/foo/bar
|
|
||||||
`)
|
|
||||||
|
|
||||||
requestType, submissionURLs = parseDiff(diff, "repositories.txt")
|
|
||||||
assert.Equal(t, "other", requestType, testName)
|
|
||||||
assert.Nil(t, submissionURLs, testName)
|
|
||||||
|
|
||||||
testName = "Submission"
|
|
||||||
diff = []byte(`
|
|
||||||
diff --git a/repositories.txt b/repositories.txt
|
|
||||||
index cff484d..9f67763 100644
|
|
||||||
--- a/repositories.txt
|
|
||||||
+++ b/repositories.txt
|
|
||||||
@@ -8,0 +9,2 @@ https://github.com/arduino-libraries/Ethernet
|
|
||||||
+https://github.com/foo/bar
|
|
||||||
+https://github.com/foo/baz
|
|
||||||
`)
|
|
||||||
|
|
||||||
requestType, submissionURLs = parseDiff(diff, "repositories.txt")
|
|
||||||
assert.Equal(t, "submission", requestType, testName)
|
|
||||||
assert.ElementsMatch(t, submissionURLs, []string{"https://github.com/foo/bar", "https://github.com/foo/baz"}, testName)
|
|
||||||
|
|
||||||
testName = "Submission w/ no newline at end of file"
|
|
||||||
diff = []byte(`
|
|
||||||
diff --git a/repositories.txt b/repositories.txt
|
|
||||||
index cff484d..1b0b80b 100644
|
|
||||||
--- a/repositories.txt
|
|
||||||
+++ b/repositories.txt
|
|
||||||
@@ -3391,0 +3392 @@ https://github.com/lbernstone/plotutils
|
|
||||||
+https://github.com/foo/bar
|
|
||||||
\ No newline at end of file
|
|
||||||
`)
|
|
||||||
|
|
||||||
requestType, submissionURLs = parseDiff(diff, "repositories.txt")
|
|
||||||
assert.Equal(t, "submission", requestType, testName)
|
|
||||||
assert.ElementsMatch(t, submissionURLs, []string{"https://github.com/foo/bar"}, testName)
|
|
||||||
|
|
||||||
testName = "Submission w/ blank line"
|
|
||||||
diff = []byte(`
|
|
||||||
diff --git a/repositories.txt b/repositories.txt
|
|
||||||
index cff484d..1b0b80b 100644
|
|
||||||
--- a/repositories.txt
|
|
||||||
+++ b/repositories.txt
|
|
||||||
@@ -3391,0 +3392 @@ https://github.com/lbernstone/plotutils
|
|
||||||
+https://github.com/foo/bar
|
|
||||||
\ No newline at end of file
|
|
||||||
`)
|
|
||||||
|
|
||||||
requestType, submissionURLs = parseDiff(diff, "repositories.txt")
|
|
||||||
assert.Equal(t, "submission", requestType, testName)
|
|
||||||
assert.ElementsMatch(t, submissionURLs, []string{"https://github.com/foo/bar"}, testName)
|
|
||||||
|
|
||||||
testName = "Removal"
|
|
||||||
diff = []byte(`
|
|
||||||
diff --git a/repositories.txt b/repositories.txt
|
|
||||||
index cff484d..38e11d8 100644
|
|
||||||
--- a/repositories.txt
|
|
||||||
+++ b/repositories.txt
|
|
||||||
@@ -8 +7,0 @@ https://github.com/firmata/arduino
|
|
||||||
-https://github.com/arduino-libraries/Ethernet
|
|
||||||
`)
|
|
||||||
|
|
||||||
requestType, submissionURLs = parseDiff(diff, "repositories.txt")
|
|
||||||
assert.Equal(t, "removal", requestType, testName)
|
|
||||||
assert.Nil(t, submissionURLs, testName)
|
|
||||||
|
|
||||||
testName = "Modification"
|
|
||||||
diff = []byte(`
|
|
||||||
diff --git a/repositories.txt b/repositories.txt
|
|
||||||
index cff484d..8b401a1 100644
|
|
||||||
--- a/repositories.txt
|
|
||||||
+++ b/repositories.txt
|
|
||||||
@@ -8 +8 @@ https://github.com/firmata/arduino
|
|
||||||
-https://github.com/arduino-libraries/Ethernet
|
|
||||||
+https://github.com/foo/bar
|
|
||||||
`)
|
|
||||||
|
|
||||||
requestType, submissionURLs = parseDiff(diff, "repositories.txt")
|
|
||||||
assert.Equal(t, "modification", requestType, testName)
|
|
||||||
assert.Equal(t, submissionURLs, []string{"https://github.com/foo/bar"}, testName)
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_normalizeURL(t *testing.T) {
|
|
||||||
testTables := []struct {
|
|
||||||
testName string
|
|
||||||
rawURL string
|
|
||||||
expectedNormalizedURL string
|
|
||||||
}{
|
|
||||||
{"Trailing slash", "https://github.com/foo/bar/", "https://github.com/foo/bar.git"},
|
|
||||||
{".git suffix", "https://github.com/foo/bar.git", "https://github.com/foo/bar.git"},
|
|
||||||
{"http://", "http://github.com/foo/bar", "https://github.com/foo/bar.git"},
|
|
||||||
{"git://", "git://github.com/foo/bar", "https://github.com/foo/bar.git"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, testTable := range testTables {
|
|
||||||
rawURL, err := url.Parse(testTable.rawURL)
|
|
||||||
require.Nil(t, err)
|
|
||||||
expectedNormalizedURL, err := url.Parse(testTable.expectedNormalizedURL)
|
|
||||||
require.Nil(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, *expectedNormalizedURL, normalizeURL(rawURL), testTable.testName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test_uRLIsUnder(t *testing.T) {
|
|
||||||
testTables := []struct {
|
|
||||||
testName string
|
|
||||||
childURL string
|
|
||||||
parentCandidates []string
|
|
||||||
assertion assert.BoolAssertionFunc
|
|
||||||
}{
|
|
||||||
{"Match, root path", "https://github.com/foo/bar", []string{"example.com", "github.com"}, assert.True},
|
|
||||||
{"Mismatch, root path", "https://github.com/foo/bar", []string{"example.com", "example.org"}, assert.False},
|
|
||||||
{"Match, subfolder", "https://github.com/foo/bar", []string{"example.com/foo", "github.com/foo"}, assert.True},
|
|
||||||
{"Mismatch, subfolder", "https://github.com/foo/bar", []string{"example.com/foo", "github.org/bar"}, assert.False},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, testTable := range testTables {
|
|
||||||
childURL, err := url.Parse(testTable.childURL)
|
|
||||||
require.Nil(t, err)
|
|
||||||
|
|
||||||
t.Run(testTable.testName, func(t *testing.T) {
|
|
||||||
testTable.assertion(t, uRLIsUnder(*childURL, testTable.parentCandidates))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
Reference in New Issue
Block a user