Update On Tue Aug 6 20:32:53 CEST 2024

This commit is contained in:
github-action[bot]
2024-08-06 20:32:53 +02:00
parent abb7c9abe3
commit 4516aba8c6
127 changed files with 1382 additions and 970 deletions

View File

@@ -61,7 +61,7 @@ jobs:
- name: Verify mmdb files
run: |
cd ./output/maxmind || exit 1
cd ./output || exit 1
go install -v github.com/maxmind/mmdbverify@latest
for name in $(ls *.mmdb); do
$(go env GOPATH)/bin/mmdbverify -file ${name}
@@ -69,14 +69,14 @@ jobs:
- name: Generate sha256 checksum for dat files
run: |
cd ./output/dat || exit 1
cd ./output || exit 1
for name in $(ls *.dat); do
sha256sum ${name} > ./${name}.sha256sum
done
- name: Generate sha256 checksum for mmdb files
run: |
cd ./output/maxmind || exit 1
cd ./output || exit 1
for name in $(ls *.mmdb); do
sha256sum ${name} > ./${name}.sha256sum
done
@@ -85,21 +85,14 @@ jobs:
env:
LICENSE_KEY: ${{ secrets.MAXMIND_GEOLITE2_LICENSE }}
run: |
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&license_key=${LICENSE_KEY}&suffix=tar.gz" -o GeoLite2-ASN.tar.gz
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o GeoLite2-ASN-CSV.zip
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${LICENSE_KEY}&suffix=tar.gz" -o GeoLite2-Country.tar.gz
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o GeoLite2-Country-CSV.zip
- name: Move files to publish directory
run: |
mkdir -p publish
mv ./output/dat/*.dat ./output/dat/*.sha256sum ./output/maxmind/*.mmdb ./output/maxmind/*.sha256sum *.gz *.zip ./publish/
cp -fpPR ./output/text ./publish
cp -fpPR ./output/srs ./publish
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&license_key=${LICENSE_KEY}&suffix=tar.gz" -o ./output/GeoLite2-ASN.tar.gz
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o ./output/GeoLite2-ASN-CSV.zip
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${LICENSE_KEY}&suffix=tar.gz" -o ./output/GeoLite2-Country.tar.gz
curl -L "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o ./output/GeoLite2-Country-CSV.zip
- name: Git push assets to "release" branch
run: |
cd publish || exit 1
cd output || exit 1
git init
git config --local user.name "github-actions[bot]"
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
@@ -111,18 +104,19 @@ jobs:
- name: Purge jsdelivr CDN
run: |
cd publish || exit 1
cd output || exit 1
for file in $(ls); do
curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@release/${file}"
done
- name: Remove some files to avoid publishing to GitHub release
run: rm -rf ./publish/*.{gz,zip} ./publish/text ./publish/srs
run: rm -rf ./output/*.{gz,zip} ./output/{clash,dat,nginx,srs,surge,text}
- name: Upload files to GitHub release
uses: svenstaro/upload-release-action@v2
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
file_glob: true
file: ./publish/*
file: ./output/*
release_name: ${{ env.RELEASE_NAME }}
tag: ${{ env.TAG_NAME }}

View File

@@ -104,6 +104,19 @@
"onlyIPType": "ipv4"
}
},
{
"type": "text",
"action": "add",
"args": {
"inputDir": "./data",
"onlyIPType": "ipv4",
"removePrefixesInLine": [
"iptables -A INPUT -s",
"iptables -A INPUT -d"
],
"removeSuffixesInLine": ["-j ACCEPT", "-j DROP"]
}
},
{
"type": "text",
"action": "add",
@@ -113,6 +126,26 @@
"onlyIPType": "ipv6"
}
},
{
"type": "text",
"action": "add",
"args": {
"name": "mylist",
"uri": "./an/example/dir/mycidr.txt",
"onlyIPType": "ipv6",
"removePrefixesInLine": ["allow from", "deny from"]
}
},
{
"type": "text",
"action": "add",
"args": {
"name": "mylist",
"uri": "./an/example/dir/mycidr.txt",
"onlyIPType": "ipv6",
"removeSuffixesInLine": [";", ","]
}
},
{
"type": "text",
"action": "add",
@@ -243,6 +276,43 @@
"onlyIPType": "ipv6"
}
},
{
"type": "text",
"action": "output",
"args": {
"outputDir": "./publish",
"onlyIPType": "ipv6",
"addPrefixInLine": "iptables -A INPUT -d ",
"addSuffixInLine": " -j DROP"
}
},
{
"type": "text",
"action": "output",
"args": {
"addPrefixInLine": "deny from "
}
},
{
"type": "text",
"action": "output",
"args": {
"outputDir": "./output/nginx/allow",
"outputExtension": ".conf",
"addPrefixInLine": "allow ",
"addSuffixInLine": ";"
}
},
{
"type": "text",
"action": "output",
"args": {
"outputDir": "./output/nginx/deny",
"outputExtension": ".conf",
"addPrefixInLine": "deny ",
"addSuffixInLine": ";"
}
},
{
"type": "stdout",
"action": "output"
@@ -292,6 +362,13 @@
"wantedList": ["cn", "private", "test"],
"onlyIPType": "ipv4"
}
},
{
"type": "surgeRuleSet",
"action": "output",
"args": {
"addSuffixInLine": ",no-resolve"
}
}
]
}

View File

@@ -85,6 +85,7 @@
"type": "v2rayGeoIPDat",
"action": "output",
"args": {
"outputDir": "./output",
"outputName": "geoip.dat"
}
},
@@ -92,6 +93,7 @@
"type": "v2rayGeoIPDat",
"action": "output",
"args": {
"outputDir": "./output",
"outputName": "geoip-only-cn-private.dat",
"wantedList": ["cn", "private"]
}
@@ -100,6 +102,7 @@
"type": "v2rayGeoIPDat",
"action": "output",
"args": {
"outputDir": "./output",
"outputName": "geoip-asn.dat",
"wantedList": [
"cloudflare",
@@ -117,14 +120,23 @@
"type": "v2rayGeoIPDat",
"action": "output",
"args": {
"outputDir": "./output",
"oneFilePerList": true,
"wantedList": ["cn", "private"]
}
},
{
"type": "v2rayGeoIPDat",
"action": "output",
"args": {
"oneFilePerList": true
}
},
{
"type": "maxmindMMDB",
"action": "output",
"args": {
"outputDir": "./output",
"outputName": "Country.mmdb",
"overwriteList": [
"cn",
@@ -144,6 +156,7 @@
"type": "maxmindMMDB",
"action": "output",
"args": {
"outputDir": "./output",
"outputName": "Country-only-cn-private.mmdb",
"wantedList": ["cn", "private"]
}
@@ -152,6 +165,7 @@
"type": "maxmindMMDB",
"action": "output",
"args": {
"outputDir": "./output",
"outputName": "Country-asn.mmdb",
"wantedList": [
"cloudflare",
@@ -172,6 +186,41 @@
{
"type": "text",
"action": "output"
},
{
"type": "text",
"action": "output",
"args": {
"outputDir": "./output/nginx/allow",
"outputExtension": ".conf",
"addPrefixInLine": "allow ",
"addSuffixInLine": ";"
}
},
{
"type": "text",
"action": "output",
"args": {
"outputDir": "./output/nginx/deny",
"outputExtension": ".conf",
"addPrefixInLine": "deny ",
"addSuffixInLine": ";"
}
},
{
"type": "clashRuleSetClassical",
"action": "output"
},
{
"type": "clashRuleSet",
"action": "output"
},
{
"type": "surgeRuleSet",
"action": "output",
"args": {
"addSuffixInLine": ",no-resolve"
}
}
]
}

View File

@@ -7,6 +7,7 @@ import (
"net"
"os"
"path/filepath"
"slices"
"strings"
"github.com/Loyalsoldier/geoip/lib"
@@ -159,7 +160,7 @@ func (m *mmdbOut) getEntryNameListInOrder(container lib.Container) []string {
}
}
list := make([]string, 0, 200)
list := make([]string, 0, 300)
for entry := range container.Loop() {
name := entry.GetName()
_, found := overwriteMap[name]
@@ -169,6 +170,9 @@ func (m *mmdbOut) getEntryNameListInOrder(container lib.Container) []string {
list = append(list, name)
}
// Sort the lists
slices.Sort(list)
// Make sure the names in overwriteList are written at last
list = append(list, overwriteList...)

View File

@@ -17,6 +17,9 @@ type textIn struct {
URI string
InputDir string
OnlyIPType lib.IPType
RemovePrefixesInLine []string
RemoveSuffixesInLine []string
}
func (t *textIn) scanFile(reader io.Reader, entry *lib.Entry) error {
@@ -40,10 +43,8 @@ func (t *textIn) scanFile(reader io.Reader, entry *lib.Entry) error {
func (t *textIn) scanFileForTextIn(reader io.Reader, entry *lib.Entry) error {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
line := scanner.Text()
line, _, _ = strings.Cut(line, "#")
line, _, _ = strings.Cut(line, "//")
line, _, _ = strings.Cut(line, "/*")
@@ -51,6 +52,19 @@ func (t *textIn) scanFileForTextIn(reader io.Reader, entry *lib.Entry) error {
if line == "" {
continue
}
line = strings.ToLower(line)
for _, prefix := range t.RemovePrefixesInLine {
line = strings.TrimSpace(strings.TrimPrefix(line, strings.ToLower(strings.TrimSpace(prefix))))
}
for _, suffix := range t.RemoveSuffixesInLine {
line = strings.TrimSpace(strings.TrimSuffix(line, strings.ToLower(strings.TrimSpace(suffix))))
}
line = strings.TrimSpace(line)
if line == "" {
continue
}
if err := entry.AddPrefix(line); err != nil {
return err
}
@@ -110,9 +124,17 @@ func (t *textIn) scanFileForClashClassicalRuleSetIn(reader io.Reader, entry *lib
continue
}
// Examples:
// IP-CIDR,162.208.16.0/24
// IP-CIDR6,2a0b:e40:1::/48
// IP-CIDR,162.208.16.0/24,no-resolve
// IP-CIDR6,2a0b:e40:1::/48,no-resolve
if strings.HasPrefix(line, "ip-cidr,") || strings.HasPrefix(line, "ip-cidr6,") {
_, line, _ = strings.Cut(line, ",")
line = strings.TrimSpace(line)
parts := strings.Split(line, ",")
if len(parts) < 2 {
continue
}
line = strings.TrimSpace(parts[1])
if line == "" {
continue
}
@@ -128,17 +150,27 @@ func (t *textIn) scanFileForClashClassicalRuleSetIn(reader io.Reader, entry *lib
func (t *textIn) scanFileForSurgeRuleSetIn(reader io.Reader, entry *lib.Entry) error {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := strings.ToLower(strings.TrimSpace(scanner.Text()))
line := scanner.Text()
line, _, _ = strings.Cut(line, "#")
line, _, _ = strings.Cut(line, "//")
line, _, _ = strings.Cut(line, "/*")
line = strings.ToLower(strings.TrimSpace(line))
if line == "" {
continue
}
// Examples:
// IP-CIDR,162.208.16.0/24
// IP-CIDR6,2a0b:e40:1::/48
// IP-CIDR,162.208.16.0/24,no-resolve
// IP-CIDR6,2a0b:e40:1::/48,no-resolve
if strings.HasPrefix(line, "ip-cidr,") || strings.HasPrefix(line, "ip-cidr6,") {
line, _, _ = strings.Cut(line, "#")
line, _, _ = strings.Cut(line, "//")
line, _, _ = strings.Cut(line, "/*")
_, line, _ = strings.Cut(line, ",")
line = strings.TrimSpace(line)
parts := strings.Split(line, ",")
if len(parts) < 2 {
continue
}
line = strings.TrimSpace(parts[1])
if line == "" {
continue
}

View File

@@ -23,15 +23,23 @@ type textOut struct {
Action lib.Action
Description string
OutputDir string
OutputExt string
Want []string
OnlyIPType lib.IPType
AddPrefixInLine string
AddSuffixInLine string
}
func newTextOut(iType string, action lib.Action, data json.RawMessage) (lib.OutputConverter, error) {
var tmp struct {
OutputDir string `json:"outputDir"`
OutputExt string `json:"outputExtension"`
Want []string `json:"wantedList"`
OnlyIPType lib.IPType `json:"onlyIPType"`
AddPrefixInLine string `json:"addPrefixInLine"`
AddSuffixInLine string `json:"addSuffixInLine"`
}
if len(data) > 0 {
@@ -53,13 +61,21 @@ func newTextOut(iType string, action lib.Action, data json.RawMessage) (lib.Outp
}
}
if tmp.OutputExt == "" {
tmp.OutputExt = ".txt"
}
return &textOut{
Type: iType,
Action: action,
Description: descTextOut,
OutputDir: tmp.OutputDir,
OutputExt: tmp.OutputExt,
Want: tmp.Want,
OnlyIPType: tmp.OnlyIPType,
AddPrefixInLine: tmp.AddPrefixInLine,
AddSuffixInLine: tmp.AddSuffixInLine,
}, nil
}
@@ -101,7 +117,13 @@ func (t *textOut) marshalBytes(entry *lib.Entry) ([]byte, error) {
func (t *textOut) marshalBytesForTextOut(buf *bytes.Buffer, entryCidr []string) error {
for _, cidr := range entryCidr {
if t.AddPrefixInLine != "" {
buf.WriteString(t.AddPrefixInLine)
}
buf.WriteString(cidr)
if t.AddSuffixInLine != "" {
buf.WriteString(t.AddSuffixInLine)
}
buf.WriteString("\n")
}
return nil
@@ -149,6 +171,9 @@ func (t *textOut) marshalBytesForSurgeRuleSetOut(buf *bytes.Buffer, entryCidr []
buf.WriteString("IP-CIDR6,")
}
buf.WriteString(cidr)
if t.AddSuffixInLine != "" {
buf.WriteString(t.AddSuffixInLine)
}
buf.WriteString("\n")
}

View File

@@ -32,6 +32,9 @@ func newTextIn(iType string, action lib.Action, data json.RawMessage) (lib.Input
URI string `json:"uri"`
InputDir string `json:"inputDir"`
OnlyIPType lib.IPType `json:"onlyIPType"`
RemovePrefixesInLine []string `json:"removePrefixesInLine"`
RemoveSuffixesInLine []string `json:"removeSuffixesInLine"`
}
if strings.TrimSpace(iType) == "" {
@@ -60,6 +63,9 @@ func newTextIn(iType string, action lib.Action, data json.RawMessage) (lib.Input
URI: tmp.URI,
InputDir: tmp.InputDir,
OnlyIPType: tmp.OnlyIPType,
RemovePrefixesInLine: tmp.RemovePrefixesInLine,
RemoveSuffixesInLine: tmp.RemoveSuffixesInLine,
}, nil
}
@@ -147,28 +153,31 @@ func (t *textIn) walkDir(dir string, entries map[string]*lib.Entry) error {
}
func (t *textIn) walkLocalFile(path, name string, entries map[string]*lib.Entry) error {
entryName := ""
name = strings.TrimSpace(name)
var filename string
if name != "" {
filename = name
entryName = name
} else {
filename = filepath.Base(path)
entryName = filepath.Base(path)
// check filename
if !regexp.MustCompile(`^[a-zA-Z0-9_.\-]+$`).MatchString(entryName) {
return fmt.Errorf("filename %s cannot be entry name, please remove special characters in it", entryName)
}
// remove file extension but not hidden files of which filename starts with "."
dotIndex := strings.LastIndex(entryName, ".")
if dotIndex > 0 {
entryName = entryName[:dotIndex]
}
}
// check filename
if !regexp.MustCompile(`^[a-zA-Z0-9_.\-]+$`).MatchString(filename) {
return fmt.Errorf("filename %s cannot be entry name, please remove special characters in it", filename)
}
dotIndex := strings.LastIndex(filename, ".")
if dotIndex > 0 {
filename = filename[:dotIndex]
entryName = strings.ToUpper(entryName)
if _, found := entries[entryName]; found {
return fmt.Errorf("found duplicated list %s", entryName)
}
if _, found := entries[filename]; found {
return fmt.Errorf("found duplicated file %s", filename)
}
entry := lib.NewEntry(filename)
entry := lib.NewEntry(entryName)
file, err := os.Open(path)
if err != nil {
return err
@@ -178,7 +187,7 @@ func (t *textIn) walkLocalFile(path, name string, entries map[string]*lib.Entry)
return err
}
entries[filename] = entry
entries[entryName] = entry
return nil
}
@@ -194,11 +203,13 @@ func (t *textIn) walkRemoteFile(url, name string, entries map[string]*lib.Entry)
return fmt.Errorf("failed to get remote file %s, http status code %d", url, resp.StatusCode)
}
name = strings.ToUpper(name)
entry := lib.NewEntry(name)
if err := t.scanFile(resp.Body, entry); err != nil {
return err
}
entries[name] = entry
return nil
}

View File

@@ -3,6 +3,7 @@ package plaintext
import (
"encoding/json"
"log"
"slices"
"strings"
"github.com/Loyalsoldier/geoip/lib"
@@ -36,28 +37,24 @@ func (t *textOut) GetDescription() string {
func (t *textOut) Output(container lib.Container) error {
// Filter want list
wantList := make(map[string]bool)
wantList := make([]string, 0, 50)
for _, want := range t.Want {
if want = strings.ToUpper(strings.TrimSpace(want)); want != "" {
wantList[want] = true
wantList = append(wantList, want)
}
}
switch len(wantList) {
case 0:
list := make([]string, 0, 300)
for entry := range container.Loop() {
data, err := t.marshalBytes(entry)
if err != nil {
return err
}
filename := strings.ToLower(entry.GetName()) + ".txt"
if err := t.writeFile(filename, data); err != nil {
return err
}
list = append(list, entry.GetName())
}
default:
for name := range wantList {
// Sort the list
slices.Sort(list)
for _, name := range list {
entry, found := container.GetEntry(name)
if !found {
log.Printf("❌ entry %s not found", name)
@@ -67,7 +64,27 @@ func (t *textOut) Output(container lib.Container) error {
if err != nil {
return err
}
filename := strings.ToLower(entry.GetName()) + ".txt"
filename := strings.ToLower(entry.GetName()) + t.OutputExt
if err := t.writeFile(filename, data); err != nil {
return err
}
}
default:
// Sort the list
slices.Sort(wantList)
for _, name := range wantList {
entry, found := container.GetEntry(name)
if !found {
log.Printf("❌ entry %s not found", name)
continue
}
data, err := t.marshalBytes(entry)
if err != nil {
return err
}
filename := strings.ToLower(entry.GetName()) + t.OutputExt
if err := t.writeFile(filename, data); err != nil {
return err
}

View File

@@ -6,6 +6,7 @@ import (
"log"
"os"
"path/filepath"
"slices"
"strings"
"github.com/Loyalsoldier/geoip/lib"
@@ -82,23 +83,39 @@ func (s *srsOut) GetDescription() string {
func (s *srsOut) Output(container lib.Container) error {
// Filter want list
wantList := make(map[string]bool)
wantList := make([]string, 0, 50)
for _, want := range s.Want {
if want = strings.ToUpper(strings.TrimSpace(want)); want != "" {
wantList[want] = true
wantList = append(wantList, want)
}
}
switch len(wantList) {
case 0:
list := make([]string, 0, 300)
for entry := range container.Loop() {
list = append(list, entry.GetName())
}
// Sort the list
slices.Sort(list)
for _, name := range list {
entry, found := container.GetEntry(name)
if !found {
log.Printf("❌ entry %s not found", name)
continue
}
if err := s.run(entry); err != nil {
return err
}
}
default:
for name := range wantList {
// Sort the list
slices.Sort(wantList)
for _, name := range wantList {
entry, found := container.GetEntry(name)
if !found {
log.Printf("❌ entry %s not found", name)

View File

@@ -94,19 +94,33 @@ func (g *geoIPDatOut) GetDescription() string {
func (g *geoIPDatOut) Output(container lib.Container) error {
// Filter want list
wantList := make(map[string]bool)
wantList := make([]string, 0, 50)
for _, want := range g.Want {
if want = strings.ToUpper(strings.TrimSpace(want)); want != "" {
wantList[want] = true
wantList = append(wantList, want)
}
}
geoIPList := new(router.GeoIPList)
geoIPList.Entry = make([]*router.GeoIP, 0, 300)
updated := false
switch len(wantList) {
case 0:
list := make([]string, 0, 300)
for entry := range container.Loop() {
list = append(list, entry.GetName())
}
// Sort the list
sort.Strings(list)
for _, name := range list {
entry, found := container.GetEntry(name)
if !found {
log.Printf("❌ entry %s not found", name)
continue
}
geoIP, err := g.generateGeoIP(entry)
if err != nil {
return err
@@ -128,7 +142,10 @@ func (g *geoIPDatOut) Output(container lib.Container) error {
}
default:
for name := range wantList {
// Sort the list
sort.Strings(wantList)
for _, name := range wantList {
entry, found := container.GetEntry(name)
if !found {
log.Printf("❌ entry %s not found", name)