Merge pull request 'Init Drone CI Build & Release' (#44) from ci/init-drone into master

Reviewed-on: https://git.unlock-music.dev/um/cli/pulls/44

.drone.jsonnet

@@ -0,0 +1,119 @@
+// generate .drone.yaml, run:
+// drone jsonnet --format --stream
+
+
+local CreateRelease() = {
+  name: 'create release',
+  image: 'plugins/gitea-release',
+  settings: {
+    api_key: { from_secret: 'GITEA_API_KEY' },
+    base_url: 'https://git.unlock-music.dev',
+    files: 'dist/*',
+    checksum: 'sha256',
+    draft: true,
+    title: '${DRONE_TAG}',
+  },
+};
+
+
+local StepGoBuild(GOOS, GOARCH) = {
+  local filepath = 'dist/um-%s-%s.tar.gz' % [GOOS, GOARCH],
+
+  name: 'go build %s/%s' % [GOOS, GOARCH],
+  image: 'golang:1.19',
+  environment: {
+    GOOS: GOOS,
+    GOARCH: GOARCH,
+  },
+  commands: [
+    'DIST_DIR=$(mktemp -d)',
+    'go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags --always)" -o $DIST_DIR ./cmd/um',
+    'mkdir -p dist',
+    'tar cz -f %s -C $DIST_DIR .' % filepath,
+  ],
+};
+
+local StepUploadArtifact(GOOS, GOARCH) = {
+  local filename = 'um-%s-%s.tar.gz' % [GOOS, GOARCH],
+  local filepath = 'dist/%s' % filename,
+  local pkgname = '${DRONE_REPO_NAME}-build',
+
+  name: 'upload artifact',
+  image: 'golang:1.19',  // reuse golang:1.19 for curl
+  environment: {
+    DRONE_GITEA_SERVER: 'https://git.unlock-music.dev',
+    GITEA_API_KEY: { from_secret: 'GITEA_API_KEY' },
+  },
+  commands: [
+    'curl --fail --include --user "um-release-bot:$GITEA_API_KEY" ' +
+    '--upload-file "%s" ' % filepath +
+    '"$DRONE_GITEA_SERVER/api/packages/${DRONE_REPO_NAMESPACE}/generic/%s/${DRONE_BUILD_NUMBER}/%s"' % [pkgname, filename],
+    'sha256sum %s' % filepath,
+    'echo $DRONE_GITEA_SERVER/${DRONE_REPO_NAMESPACE}/-/packages/generic/%s/${DRONE_BUILD_NUMBER}' % pkgname,
+  ],
+};
+
+
+local PipelineBuild(GOOS, GOARCH, RUN_TEST) = {
+  name: 'build %s/%s' % [GOOS, GOARCH],
+  kind: 'pipeline',
+  type: 'docker',
+  steps: [
+           {
+             name: 'fetch tags',
+             image: 'alpine/git',
+             commands: ['git fetch --tags'],
+           },
+         ] +
+         (
+           if RUN_TEST then [{
+             name: 'go test',
+             image: 'golang:1.19',
+             commands: ['go test -v ./...'],
+           }] else []
+         )
+         +
+         [
+           StepGoBuild(GOOS, GOARCH),
+           StepUploadArtifact(GOOS, GOARCH),
+         ],
+  trigger: {
+    event: ['push', 'pull_request'],
+  },
+};
+
+local PipelineRelease() = {
+  name: 'release',
+  kind: 'pipeline',
+  type: 'docker',
+  steps: [
+    {
+      name: 'fetch tags',
+      image: 'alpine/git',
+      commands: ['git fetch --tags'],
+    },
+    {
+      name: 'go test',
+      image: 'golang:1.19',
+      commands: ['go test -v ./...'],
+    },
+    StepGoBuild('linux', 'amd64'),
+    StepGoBuild('linux', 'arm64'),
+    StepGoBuild('linux', '386'),
+    StepGoBuild('windows', 'amd64'),
+    StepGoBuild('windows', '386'),
+    StepGoBuild('darwin', 'amd64'),
+    StepGoBuild('darwin', 'arm64'),
+    CreateRelease(),
+  ],
+  trigger: {
+    event: ['tag'],
+  },
+};
+
+[
+  PipelineBuild('linux', 'amd64', true),
+  PipelineBuild('windows', 'amd64', false),
+  PipelineBuild('darwin', 'amd64', false),
+  PipelineRelease(),
+]

.drone.yml

@@ -0,0 +1,212 @@
+---
+kind: pipeline
+name: build linux/amd64
+steps:
+- commands:
+  - git fetch --tags
+  image: alpine/git
+  name: fetch tags
+- commands:
+  - go test -v ./...
+  image: golang:1.19
+  name: go test
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-linux-amd64.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: amd64
+    GOOS: linux
+  image: golang:1.19
+  name: go build linux/amd64
+- commands:
+  - curl --fail --include --user "um-release-bot:$GITEA_API_KEY" --upload-file "dist/um-linux-amd64.tar.gz"
+    "$DRONE_GITEA_SERVER/api/packages/${DRONE_REPO_NAMESPACE}/generic/${DRONE_REPO_NAME}-build/${DRONE_BUILD_NUMBER}/um-linux-amd64.tar.gz"
+  - sha256sum dist/um-linux-amd64.tar.gz
+  - echo $DRONE_GITEA_SERVER/${DRONE_REPO_NAMESPACE}/-/packages/generic/${DRONE_REPO_NAME}-build/${DRONE_BUILD_NUMBER}
+  environment:
+    DRONE_GITEA_SERVER: https://git.unlock-music.dev
+    GITEA_API_KEY:
+      from_secret: GITEA_API_KEY
+  image: golang:1.19
+  name: upload artifact
+trigger:
+  event:
+  - push
+  - pull_request
+type: docker
+---
+kind: pipeline
+name: build windows/amd64
+steps:
+- commands:
+  - git fetch --tags
+  image: alpine/git
+  name: fetch tags
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-windows-amd64.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: amd64
+    GOOS: windows
+  image: golang:1.19
+  name: go build windows/amd64
+- commands:
+  - curl --fail --include --user "um-release-bot:$GITEA_API_KEY" --upload-file "dist/um-windows-amd64.tar.gz"
+    "$DRONE_GITEA_SERVER/api/packages/${DRONE_REPO_NAMESPACE}/generic/${DRONE_REPO_NAME}-build/${DRONE_BUILD_NUMBER}/um-windows-amd64.tar.gz"
+  - sha256sum dist/um-windows-amd64.tar.gz
+  - echo $DRONE_GITEA_SERVER/${DRONE_REPO_NAMESPACE}/-/packages/generic/${DRONE_REPO_NAME}-build/${DRONE_BUILD_NUMBER}
+  environment:
+    DRONE_GITEA_SERVER: https://git.unlock-music.dev
+    GITEA_API_KEY:
+      from_secret: GITEA_API_KEY
+  image: golang:1.19
+  name: upload artifact
+trigger:
+  event:
+  - push
+  - pull_request
+type: docker
+---
+kind: pipeline
+name: build darwin/amd64
+steps:
+- commands:
+  - git fetch --tags
+  image: alpine/git
+  name: fetch tags
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-darwin-amd64.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: amd64
+    GOOS: darwin
+  image: golang:1.19
+  name: go build darwin/amd64
+- commands:
+  - curl --fail --include --user "um-release-bot:$GITEA_API_KEY" --upload-file "dist/um-darwin-amd64.tar.gz"
+    "$DRONE_GITEA_SERVER/api/packages/${DRONE_REPO_NAMESPACE}/generic/${DRONE_REPO_NAME}-build/${DRONE_BUILD_NUMBER}/um-darwin-amd64.tar.gz"
+  - sha256sum dist/um-darwin-amd64.tar.gz
+  - echo $DRONE_GITEA_SERVER/${DRONE_REPO_NAMESPACE}/-/packages/generic/${DRONE_REPO_NAME}-build/${DRONE_BUILD_NUMBER}
+  environment:
+    DRONE_GITEA_SERVER: https://git.unlock-music.dev
+    GITEA_API_KEY:
+      from_secret: GITEA_API_KEY
+  image: golang:1.19
+  name: upload artifact
+trigger:
+  event:
+  - push
+  - pull_request
+type: docker
+---
+kind: pipeline
+name: release
+steps:
+- commands:
+  - git fetch --tags
+  image: alpine/git
+  name: fetch tags
+- commands:
+  - go test -v ./...
+  image: golang:1.19
+  name: go test
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-linux-amd64.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: amd64
+    GOOS: linux
+  image: golang:1.19
+  name: go build linux/amd64
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-linux-arm64.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: arm64
+    GOOS: linux
+  image: golang:1.19
+  name: go build linux/arm64
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-linux-386.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: "386"
+    GOOS: linux
+  image: golang:1.19
+  name: go build linux/386
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-windows-amd64.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: amd64
+    GOOS: windows
+  image: golang:1.19
+  name: go build windows/amd64
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-windows-386.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: "386"
+    GOOS: windows
+  image: golang:1.19
+  name: go build windows/386
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-darwin-amd64.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: amd64
+    GOOS: darwin
+  image: golang:1.19
+  name: go build darwin/amd64
+- commands:
+  - DIST_DIR=$(mktemp -d)
+  - go build -v -trimpath -ldflags="-w -s -X main.AppVersion=$(git describe --tags
+    --always)" -o $DIST_DIR ./cmd/um
+  - mkdir -p dist
+  - tar cz -f dist/um-darwin-arm64.tar.gz -C $DIST_DIR .
+  environment:
+    GOARCH: arm64
+    GOOS: darwin
+  image: golang:1.19
+  name: go build darwin/arm64
+- image: plugins/gitea-release
+  name: create release
+  settings:
+    api_key:
+      from_secret: GITEA_API_KEY
+    base_url: https://git.unlock-music.dev
+    checksum: sha256
+    draft: true
+    files: dist/*
+    title: ${DRONE_TAG}
+trigger:
+  event:
+  - tag
+type: docker

algo/common/raw.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+
+	"unlock-music.dev/cli/internal/sniff"
 )
 
 type RawDecoder struct {
@@ -26,7 +28,7 @@ func (d *RawDecoder) Validate() error {
 	}
 
 	var ok bool
-	d.audioExt, ok = SniffAll(header)
+	d.audioExt, ok = sniff.AudioExtension(header)
 	if !ok {
 		return errors.New("raw: sniff audio type failed")
 	}

algo/common/sniff.go

@@ -1,55 +0,0 @@
-package common
-
-import "bytes"
-
-type Sniffer func(header []byte) bool
-
-var snifferRegistry = map[string]Sniffer{
-	".mp3":  SnifferMP3,
-	".flac": SnifferFLAC,
-	".ogg":  SnifferOGG,
-	".m4a":  SnifferM4A,
-	".wav":  SnifferWAV,
-	".wma":  SnifferWMA,
-	".aac":  SnifferAAC,
-	".dff":  SnifferDFF,
-}
-
-func SniffAll(header []byte) (string, bool) {
-	for ext, sniffer := range snifferRegistry {
-		if sniffer(header) {
-			return ext, true
-		}
-	}
-	return "", false
-}
-
-func SnifferM4A(header []byte) bool {
-	return len(header) >= 8 && bytes.Equal([]byte("ftyp"), header[4:8])
-}
-
-func SnifferOGG(header []byte) bool {
-	return bytes.HasPrefix(header, []byte("OggS"))
-}
-
-func SnifferFLAC(header []byte) bool {
-	return bytes.HasPrefix(header, []byte("fLaC"))
-}
-func SnifferMP3(header []byte) bool {
-	return bytes.HasPrefix(header, []byte("ID3"))
-}
-func SnifferWAV(header []byte) bool {
-	return bytes.HasPrefix(header, []byte("RIFF"))
-}
-func SnifferWMA(header []byte) bool {
-	return bytes.HasPrefix(header, []byte("\x30\x26\xb2\x75\x8e\x66\xcf\x11\xa6\xd9\x00\xaa\x00\x62\xce\x6c"))
-}
-func SnifferAAC(header []byte) bool {
-	return bytes.HasPrefix(header, []byte{0xFF, 0xF1})
-}
-
-// SnifferDFF sniff a DSDIFF format
-// reference to: https://www.sonicstudio.com/pdf/dsd/DSDIFF_1.5_Spec.pdf
-func SnifferDFF(header []byte) bool {
-	return bytes.HasPrefix(header, []byte("FRM8"))
-}

algo/ncm/ncm.go

@@ -206,8 +206,12 @@ func (d *Decoder) GetCoverImage(ctx context.Context) ([]byte, error) {
 	if d.cover != nil {
 		return d.cover, nil
 	}
+
+	if d.meta == nil {
+		return nil, errors.New("ncm meta not found")
+	}
 	imgURL := d.meta.GetAlbumImageURL()
-	if d.meta != nil && !strings.HasPrefix(imgURL, "http") {
+	if !strings.HasPrefix(imgURL, "http") {
 		return nil, nil // no cover image
 	}
 
@@ -215,18 +219,19 @@ func (d *Decoder) GetCoverImage(ctx context.Context) ([]byte, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, imgURL, nil)
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return nil, fmt.Errorf("download image failed: %w", err)
+		return nil, fmt.Errorf("ncm download image failed: %w", err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("download image failed: unexpected http status %s", resp.Status)
+		return nil, fmt.Errorf("ncm download image failed: unexpected http status %s", resp.Status)
 	}
-	data, err := io.ReadAll(resp.Body)
+	d.cover, err = io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("download image failed: %w", err)
+		return nil, fmt.Errorf("ncm download image failed: %w", err)
 	}
-	return data, nil
+
+	return d.cover, nil
 }
 
 func (d *Decoder) GetMeta() common.Meta {

algo/qmc/qmc.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"unlock-music.dev/cli/algo/common"
+	"unlock-music.dev/cli/internal/sniff"
 )
 
 type Decoder struct {
@@ -89,7 +90,7 @@ func (d *Decoder) validateDecode() error {
 	}
 
 	d.cipher.Decrypt(buf, 0)
-	_, ok := common.SniffAll(buf)
+	_, ok := sniff.AudioExtension(buf)
 	if !ok {
 		return errors.New("qmc: detect file type failed")
 	}

algo/tm/tm.go

@@ -7,6 +7,7 @@ import (
 	"io"
 
 	"unlock-music.dev/cli/algo/common"
+	"unlock-music.dev/cli/internal/sniff"
 )
 
 var replaceHeader = []byte{0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70}
@@ -30,7 +31,7 @@ func (d *Decoder) Validate() error {
 		return nil
 	}
 
-	if _, ok := common.SniffAll(header); ok { // not encrypted
+	if _, ok := sniff.AudioExtension(header); ok { // not encrypted
 		d.audio = io.MultiReader(bytes.NewReader(header), d.raw)
 		return nil
 	}

algo/ximalaya/x2m_crypto.go

@@ -15,10 +15,10 @@ var x2mScrambleTableBytes []byte
 
 func init() {
 	if len(x2mScrambleTableBytes) != 2*x2mHeaderSize {
-		panic("invalid x3m scramble table")
+		panic("invalid x2m scramble table")
 	}
-	for i := range x3mScrambleTable {
+	for i := range x2mScrambleTable {
-		x3mScrambleTable[i] = binary.LittleEndian.Uint16(x2mScrambleTableBytes[i*2:])
+		x2mScrambleTable[i] = binary.LittleEndian.Uint16(x2mScrambleTableBytes[i*2:])
 	}
 }
 

algo/ximalaya/ximalaya.go

@@ -6,6 +6,7 @@ import (
 	"io"
 
 	"unlock-music.dev/cli/algo/common"
+	"unlock-music.dev/cli/internal/sniff"
 )
 
 type Decoder struct {
@@ -27,7 +28,7 @@ func (d *Decoder) Validate() error {
 
 	{ // try to decode with x2m
 		header := decryptX2MHeader(encryptedHeader)
-		if _, ok := common.SniffAll(header); ok {
+		if _, ok := sniff.AudioExtension(header); ok {
 			d.audio = io.MultiReader(bytes.NewReader(header), d.rd)
 			return nil
 		}
@@ -36,7 +37,7 @@ func (d *Decoder) Validate() error {
 	{ // try to decode with x3m
 		// not read file again, since x2m and x3m have the same header size
 		header := decryptX3MHeader(encryptedHeader)
-		if _, ok := common.SniffAll(header); ok {
+		if _, ok := sniff.AudioExtension(header); ok {
 			d.audio = io.MultiReader(bytes.NewReader(header), d.rd)
 			return nil
 		}

cmd/um/main.go

@@ -25,6 +25,7 @@ import (
 	_ "unlock-music.dev/cli/algo/xiami"
 	_ "unlock-music.dev/cli/algo/ximalaya"
 	"unlock-music.dev/cli/internal/logging"
+	"unlock-music.dev/cli/internal/sniff"
 )
 
 var AppVersion = "v0.0.6"
@@ -182,13 +183,10 @@ func tryDecFile(inputFile string, outputDir string, allDec []common.NewDecoderFu
 		return fmt.Errorf("read header failed: %w", err)
 	}
 
-	outExt := ".mp3"
+	outExt := sniff.AudioExtensionWithFallback(header.Bytes(), ".mp3")
-	if ext, ok := common.SniffAll(header.Bytes()); ok {
+	inFilename := strings.TrimSuffix(filepath.Base(inputFile), filepath.Ext(inputFile))
-		outExt = ext
-	}
-	filenameOnly := strings.TrimSuffix(filepath.Base(inputFile), filepath.Ext(inputFile))
 
-	outPath := filepath.Join(outputDir, filenameOnly+outExt)
+	outPath := filepath.Join(outputDir, inFilename+outExt)
 	outFile, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
 	if err != nil {
 		return err

internal/sniff/audio.go

@@ -0,0 +1,61 @@
+package sniff
+
+import "bytes"
+
+type Sniffer interface {
+	Sniff(header []byte) bool
+}
+
+var audioExtensions = map[string]Sniffer{
+	// ref: https://mimesniff.spec.whatwg.org
+	".mp3": prefixSniffer("ID3"),
+	".ogg": prefixSniffer("OggS"),
+	".wav": prefixSniffer("RIFF"),
+
+	// ref: https://www.loc.gov/preservation/digital/formats/fdd/fdd000027.shtml
+	".wma": prefixSniffer{
+		0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11,
+		0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c,
+	},
+
+	// ref: https://www.garykessler.net/library/file_sigs.html
+	".m4a": mpeg4Sniffer{},            // MPEG-4 container, m4a treat as audio
+	".aac": prefixSniffer{0xFF, 0xF1}, // MPEG-4 AAC-LC
+
+	".flac": prefixSniffer("fLaC"), // ref: https://xiph.org/flac/format.html
+	".dff":  prefixSniffer("FRM8"), // DSDIFF, ref: https://www.sonicstudio.com/pdf/dsd/DSDIFF_1.5_Spec.pdf
+
+}
+
+// AudioExtension sniffs the known audio types, and returns the file extension.
+// header is recommended to at least 16 bytes.
+func AudioExtension(header []byte) (string, bool) {
+	for ext, sniffer := range audioExtensions {
+		if sniffer.Sniff(header) {
+			return ext, true
+		}
+	}
+	return "", false
+}
+
+// AudioExtensionWithFallback is equivalent to AudioExtension, but returns fallback
+// most likely to use .mp3 as fallback, because mp3 files may not have ID3v2 tag.
+func AudioExtensionWithFallback(header []byte, fallback string) string {
+	ext, ok := AudioExtension(header)
+	if !ok {
+		return fallback
+	}
+	return ext
+}
+
+type prefixSniffer []byte
+
+func (s prefixSniffer) Sniff(header []byte) bool {
+	return bytes.HasPrefix(header, s)
+}
+
+type mpeg4Sniffer struct{}
+
+func (mpeg4Sniffer) Sniff(header []byte) bool {
+	return len(header) >= 8 && bytes.Equal([]byte("ftyp"), header[4:8])
+}