diff --git a/.github/update.log b/.github/update.log index e7fa5a1e77..27e00dcb61 100644 --- a/.github/update.log +++ b/.github/update.log @@ -658,3 +658,4 @@ Update On Wed May 22 20:31:05 CEST 2024 Update On Thu May 23 20:29:12 CEST 2024 Update On Fri May 24 20:29:24 CEST 2024 Update On Sat May 25 20:28:13 CEST 2024 +Update On Sun May 26 20:29:29 CEST 2024 diff --git a/clash-nyanpasu/backend/Cargo.lock b/clash-nyanpasu/backend/Cargo.lock index 81acbfd429..5076554da9 100644 --- a/clash-nyanpasu/backend/Cargo.lock +++ b/clash-nyanpasu/backend/Cargo.lock @@ -932,7 +932,7 @@ dependencies = [ "windows-sys 0.52.0", "winreg 0.52.0", "wry", - "zip 2.0.0", + "zip 2.1.0", "zip-extensions", ] @@ -5109,18 +5109,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.202" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", @@ -7833,9 +7833,9 @@ dependencies = [ [[package]] name = "zip" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fccb210625924ecbbe92f9bb497d04b167b64fe5540cec75f10b16e0c51ee92b" +checksum = "e2568cd0f20e86cd9a7349fe05178f7bd22f22724678448ae5a9bac266df2689" dependencies = [ "aes", "arbitrary", @@ -7849,6 +7849,7 @@ dependencies = [ "hmac", "indexmap 2.2.6", "lzma-rs", + "memchr", "pbkdf2 0.12.2", "rand 0.8.5", "sha1", diff --git a/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/profile-item.tsx b/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/profile-item.tsx index 411f701866..913da4ddb9 100644 --- a/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/profile-item.tsx +++ b/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/profile-item.tsx @@ -4,6 +4,7 @@ import { FilterDrama, InsertDriveFile, FiberManualRecord, + Terminal, } from "@mui/icons-material"; import LoadingButton from "@mui/lab/LoadingButton"; import { @@ -14,7 +15,8 @@ import { Menu, MenuItem, useTheme, - lighten, + Button, + alpha, } from "@mui/material"; import { Profile, useClash } from "@nyanpasu/interface"; import dayjs from "dayjs"; @@ -27,11 +29,15 @@ import { useLockFn, useSetState } from "ahooks"; export interface ProfileItemProps { item: Profile.Item; selected?: boolean; + onClickChains: (item: Profile.Item) => void; + chainsSelected?: boolean; } export const ProfileItem = memo(function ProfileItem({ item, selected, + onClickChains, + chainsSelected, }: ProfileItemProps) { const { t } = useTranslation(); @@ -141,6 +147,7 @@ export const ProfileItem = memo(function ProfileItem({ const menuMapping = { Select: () => handleSelect(), Edit: () => setOpen(true), + Chains: () => onClickChains(item), "Open File": () => viewProfile(item.uid), Update: () => handleUpdate(), "Update(Proxy)": () => handleUpdate(true), @@ -156,7 +163,7 @@ export const ProfileItem = memo(function ProfileItem({ sx={{ borderRadius: 6, backgroundColor: selected - ? lighten(palette.primary.main, 0.9) + ? alpha(palette.primary.main, 0.2) : undefined, }} > @@ -201,6 +208,16 @@ export const ProfileItem = memo(function ProfileItem({ )}
+ + {isRemote && ( instanceRef.current?.getValue(), })); + useUpdateEffect(() => { + if (!language) return; + + monaco.editor.setModelLanguage(instanceRef.current!.getModel()!, language); + + console.log(language, instanceRef.current?.getModel()?.getLanguageId()); + }, [language]); + return open &&
; }); diff --git a/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/profile-side.tsx b/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/profile-side.tsx new file mode 100644 index 0000000000..14b3139c9a --- /dev/null +++ b/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/profile-side.tsx @@ -0,0 +1,200 @@ +import { Add, Close, Edit, RamenDining, Terminal } from "@mui/icons-material"; +import { + Divider, + ListItemButton, + ListItemIcon, + ListItemText, + IconButton, + List, + useTheme, + alpha, +} from "@mui/material"; +import { Profile, useClash } from "@nyanpasu/interface"; +import { useState } from "react"; +import { filterProfiles } from "./utils"; +import { useLockFn } from "ahooks"; +import { Expand, ExpandMore } from "@nyanpasu/ui"; +import { isEmpty } from "lodash-es"; +import { ScriptDialog } from "./script-dialog"; +import { VList } from "virtua"; + +export interface ProfileSideProps { + profile?: Profile.Item; + global?: boolean; + onClose: () => void; +} + +export const ProfileSide = ({ profile, global, onClose }: ProfileSideProps) => { + const { palette } = useTheme(); + + const [open, setOpen] = useState(false); + + const { getProfiles, setProfilesConfig, getRuntimeLogs, setProfiles } = + useClash(); + + const { scripts } = filterProfiles(getProfiles.data?.items); + + const handleChainClick = useLockFn(async (uid: string) => { + const chains = global + ? getProfiles.data?.chain ?? [] + : profile?.chains ?? []; + + const updatedChains = chains.includes(uid) + ? chains.filter((chain) => chain !== uid) + : [...chains, uid]; + + if (global) { + await setProfilesConfig({ chain: updatedChains }); + } else { + await setProfiles(uid, { chains: updatedChains }); + } + }); + + const [item, setItem] = useState(); + + const handleEditChain = async (item: Profile.Item) => { + setItem(item); + setOpen(true); + }; + + const [expand, setExpand] = useState(false); + + return ( +
+
+
+
+
Proxy Chains
+ +
+ {global ? "Global Chain" : profile?.name} +
+
+ + + + +
+ + + {scripts?.map((item, index) => { + const selected = global + ? getProfiles.data?.chain?.includes(item.uid) + : profile?.chains?.includes(item.uid); + + return ( + handleChainClick(item.uid)} + > + + + { + e.preventDefault(); + e.stopPropagation(); + handleEditChain(item); + }} + > + + + + ); + })} + + setOpen(true)} + > + + + + + + + +
+ + { + setOpen(false); + setItem(undefined); + }} + /> + +
+ + +
+
+ + + Console +
+ + setExpand(!expand)} + /> +
+ + + + + + {!isEmpty(getRuntimeLogs.data) ? ( + Object.entries(getRuntimeLogs.data).map(([uid, content]) => { + return content.map((item, index) => { + const name = scripts?.find( + (script) => script.uid === uid, + )?.name; + + return ( + <> + {index !== 0 && } + +
+ [{name}]: + {item} +
+ + ); + }); + }) + ) : ( +
+ +

No Log

+
+ )} + + +
+
+ ); +}; + +export default ProfileSide; diff --git a/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/script-dialog.tsx b/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/script-dialog.tsx new file mode 100644 index 0000000000..f9e2b9b178 --- /dev/null +++ b/clash-nyanpasu/frontend/nyanpasu/src/components/profiles/script-dialog.tsx @@ -0,0 +1,199 @@ +import { Divider } from "@mui/material"; +import { BaseDialog, BaseDialogProps } from "@nyanpasu/ui"; +import { useRef } from "react"; +import { useAsyncEffect, useReactive } from "ahooks"; +import { Profile, useClash } from "@nyanpasu/interface"; +import { ProfileMonacoView, ProfileMonacoViewRef } from "./profile-monaco-view"; +import { SelectElement, TextFieldElement, useForm } from "react-hook-form-mui"; +import { useTranslation } from "react-i18next"; +import { isEqual } from "lodash-es"; + +export interface ScriptDialogProps extends Omit { + open: boolean; + onClose: () => void; + item?: Profile.Item; +} + +export const ScriptDialog = ({ + open, + item, + onClose, + ...props +}: ScriptDialogProps) => { + const { t } = useTranslation(); + + const { getProfileFile, setProfileFile, createProfile, setProfiles } = + useClash(); + + const optionTypeMapping = [ + { + id: "js", + value: { script: "javascript" }, + language: "javascript", + label: t("JavaScript"), + }, + { + id: "lua", + value: { script: "lua" }, + language: "lua", + label: t("LuaScript"), + }, + { + id: "merge", + value: "merge", + language: "yaml", + label: t("Merge"), + }, + ]; + + const preprocessing = () => { + const result = optionTypeMapping.find((option) => + isEqual(option.value, item?.type), + ); + + return { ...item, type: result?.id } as Profile.Item; + }; + + const { control, watch, handleSubmit, reset } = useForm({ + defaultValues: item + ? preprocessing() + : { + type: "merge", + chains: [], + name: "New Script", + desc: "", + }, + }); + + const profileMonacoViewRef = useRef(null); + + const editor = useReactive({ + value: "", + language: "javascript", + }); + + const handleTypeChange = () => { + const language = optionTypeMapping.find((option) => + isEqual(option.id, watch("type")), + )?.language; + + if (language) { + editor.language = language; + } + }; + + const isEdit = Boolean(item); + + const commonProps = { + autoComplete: "off", + autoCorrect: "off", + fullWidth: true, + }; + + const onSubmit = handleSubmit(async (form) => { + const value = profileMonacoViewRef.current?.getValue() || ""; + + const type = optionTypeMapping.find((option) => + isEqual(option.id, form.type), + )?.value; + + const data = { + ...form, + type, + } as Profile.Item; + + try { + if (isEdit) { + await setProfiles(data.uid, data); + + await setProfileFile(data.uid, value); + } else { + await createProfile(data, value); + } + + setTimeout(() => reset(), 300); + + onClose(); + } finally { + } + }); + + useAsyncEffect(async () => { + editor.value = await getProfileFile(item?.uid); + + if (item) { + reset(item); + } else { + reset(); + } + }, [open]); + + return ( + onClose()} + onOk={onSubmit} + divider + sx={{ + " .MuiDialog-paper": { + maxWidth: "90vw", + maxHeight: "90vh", + }, + }} + contentSx={{ + overflow: "auto", + width: "90vw", + height: "90vh", + padding: 0, + }} + {...props} + > +
+
+
+ {!isEdit && ( + handleTypeChange()} + /> + )} + + + + +
+
+ + + + +
+ + ); +}; diff --git a/clash-nyanpasu/frontend/nyanpasu/src/components/proxies/node-list.tsx b/clash-nyanpasu/frontend/nyanpasu/src/components/proxies/node-list.tsx index 634560c4f1..dc6f500db6 100644 --- a/clash-nyanpasu/frontend/nyanpasu/src/components/proxies/node-list.tsx +++ b/clash-nyanpasu/frontend/nyanpasu/src/components/proxies/node-list.tsx @@ -13,9 +13,12 @@ import { proxyGroupAtom, proxyGroupSortAtom } from "@/store"; import { CSSProperties, memo, useEffect, useMemo, useState } from "react"; import { classNames } from "@/utils"; import { VList } from "virtua"; +import { AnimatePresence, motion } from "framer-motion"; type History = Clash.Proxy["history"]; +type RenderClashProxy = Clash.Proxy & { renderLayoutKey: string }; + const filterDelay = (history?: History): number => { if (!history || history.length == 0) { return 0; @@ -218,15 +221,36 @@ export const NodeList = () => { default: 4, }); - const [renderList, setRenderList] = useState[][]>([]); + const [renderList, setRenderList] = useState([]); useEffect(() => { if (!group?.all) return; - const list = group?.all?.reduce[][]>( + const nodeNames: string[] = []; + + const list = group?.all?.reduce( (result, value, index) => { - if (index % column === 0) result.push([]); - result[Math.floor(index / column)].push(value); + const getKey = () => { + const filter = nodeNames.filter((i) => i === value.name); + + if (filter.length === 0) { + return value.name; + } else { + return `${value.name}-${filter.length}`; + } + }; + + if (index % column === 0) { + result.push([]); + } + + result[Math.floor(index / column)].push({ + ...value, + renderLayoutKey: getKey(), + }); + + nodeNames.push(value.name); + return result; }, [], @@ -240,35 +264,42 @@ export const NodeList = () => { }; return ( - - {renderList?.map((node, index) => { - return ( -
- {node.map((render, index) => { - return ( -
- hendleClick(render.name)} - onClickDelay={async () => { - await updateProxiesDelay(render.name); - }} - /> -
- ); - })} -
- ); - })} - + + + {renderList?.map((node, index) => { + return ( +
+ {node.map((render) => { + return ( + + hendleClick(render.name)} + onClickDelay={async () => { + await updateProxiesDelay(render.name); + }} + /> + + ); + })} +
+ ); + })} + + ); }; diff --git a/clash-nyanpasu/frontend/nyanpasu/src/pages/profiles.tsx b/clash-nyanpasu/frontend/nyanpasu/src/pages/profiles.tsx index b317a51370..14604a649d 100644 --- a/clash-nyanpasu/frontend/nyanpasu/src/pages/profiles.tsx +++ b/clash-nyanpasu/frontend/nyanpasu/src/pages/profiles.tsx @@ -1,48 +1,102 @@ -import { Button, alpha, useTheme } from "@mui/material"; -import { useNyanpasu } from "@nyanpasu/interface"; +import { Button } from "@mui/material"; +import { Profile, useClash } from "@nyanpasu/interface"; import { SidePage } from "@nyanpasu/ui"; +import { useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; -import Grid from "@mui/material/Unstable_Grid2"; -import { Add } from "@mui/icons-material"; import ProfileItem from "@/components/profiles/profile-item"; +import ProfileSide from "@/components/profiles/profile-side"; import { filterProfiles } from "@/components/profiles/utils"; +import NewProfileButton from "@/components/profiles/new-profile-button"; +import { QuickImport } from "@/components/profiles/quick-import"; +import Masonry from "@mui/lab/Masonry"; export const ProfilePage = () => { const { t } = useTranslation(); - const { getProfiles } = useNyanpasu(); + const { getProfiles } = useClash(); - const { profiles } = filterProfiles(getProfiles.data?.items); + const { profiles, scripts } = filterProfiles(getProfiles.data?.items); - const { palette } = useTheme(); + useEffect(() => { + console.log("getProfiles.data:", getProfiles.data); + console.log("profiles:", profiles); + console.log("scripts:", scripts); + }, [getProfiles.data?.items]); + + const [globalChain, setGlobalChain] = useState(false); + + const handleGlobalChainClick = () => { + setChainsSelected(undefined); + setGlobalChain(!globalChain); + }; + + const [chainsSelected, setChainsSelected] = useState(); + + const onClickChains = (profile: Profile.Item) => { + setGlobalChain(false); + + if (chainsSelected?.uid == profile.uid) { + setChainsSelected(undefined); + } else { + setChainsSelected(profile); + } + }; + + const handleSideClose = () => { + setChainsSelected(undefined); + setGlobalChain(false); + }; return ( - -
- - {profiles?.map((item, index) => { - return ( - - - - ); - })} - + + +
+ } + sideClassName="!overflow-visible" + side={ + (globalChain || chainsSelected) && ( + + ) + } + > +
+ + + {profiles && ( + + {profiles.map((item, index) => { + return ( + + ); + })} + + )}
- + ); }; diff --git a/clash-nyanpasu/frontend/ui/materialYou/components/expandMore/index.tsx b/clash-nyanpasu/frontend/ui/materialYou/components/expandMore/index.tsx index acecfb9e4b..aa76bd151b 100644 --- a/clash-nyanpasu/frontend/ui/materialYou/components/expandMore/index.tsx +++ b/clash-nyanpasu/frontend/ui/materialYou/components/expandMore/index.tsx @@ -4,6 +4,7 @@ import useTheme from "@mui/material/styles/useTheme"; interface ExpandMoreProps extends IconButtonProps { expand: boolean; + reverse?: boolean; } /** @@ -15,14 +16,20 @@ interface ExpandMoreProps extends IconButtonProps { * @author keiko233 * @copyright LibNyanpasu org. 2024 */ -export const ExpandMore = ({ expand, ...props }: ExpandMoreProps) => { +export const ExpandMore = ({ expand, reverse, ...props }: ExpandMoreProps) => { const { transitions } = useTheme(); return ( = ({ children, sideBar, side, + sideClassName, toolBar, noChildrenScroll, flexReverse, @@ -82,7 +84,7 @@ export const SidePage: FC = ({ {sideBar &&
{sideBar}
}
-
{side}
+
{side}
diff --git a/clash-verge-rev/UPDATELOG.md b/clash-verge-rev/UPDATELOG.md index 3934b081d2..76b524e679 100644 --- a/clash-verge-rev/UPDATELOG.md +++ b/clash-verge-rev/UPDATELOG.md @@ -1,3 +1,25 @@ +## v1.6.4 + +### Features + +- 系统代理支持 PAC 模式 +- 允许关闭不使用的端口 +- 使用新的应用图标 +- MacOS 支持切换托盘图标单色/彩色模式 +- CSS 注入支持通过编辑器编辑 +- 优化代理组列表性能 +- 优化流量图显性能 +- 支持波斯语 + +### Bugs Fixes + +- Kill 内核后 Tun 开启缓慢的问题 +- 代理绕过为空时使用默认值 +- 无法读取剪切板内容 +- Windows 下覆盖安装无法内核占用问题 + +--- + ## v1.6.2 ### Features @@ -16,6 +38,8 @@ - Linux 下与 N 卡的兼容性问题 - 修改 Tun 设置不立即生效 +--- + ## v1.6.1 ### Features diff --git a/clash-verge-rev/package.json b/clash-verge-rev/package.json index ed4e6eb68f..1b21fe1942 100644 --- a/clash-verge-rev/package.json +++ b/clash-verge-rev/package.json @@ -1,6 +1,6 @@ { "name": "clash-verge", - "version": "1.6.2", + "version": "1.6.4", "license": "GPL-3.0-only", "scripts": { "dev": "tauri dev", @@ -32,7 +32,6 @@ "@types/json-schema": "^7.0.15", "ahooks": "^3.7.11", "axios": "^1.6.8", - "axios-tauri-api-adapter": "^0.2.1", "dayjs": "1.11.5", "i18next": "^23.11.3", "lodash-es": "^4.17.21", diff --git a/clash-verge-rev/pnpm-lock.yaml b/clash-verge-rev/pnpm-lock.yaml index 845cb0c2cf..9da2056b78 100644 --- a/clash-verge-rev/pnpm-lock.yaml +++ b/clash-verge-rev/pnpm-lock.yaml @@ -49,9 +49,6 @@ importers: axios: specifier: ^1.6.8 version: 1.6.8 - axios-tauri-api-adapter: - specifier: ^0.2.1 - version: 0.2.1 dayjs: specifier: 1.11.5 version: 1.11.5 @@ -1605,13 +1602,6 @@ packages: integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==, } - axios-tauri-api-adapter@0.2.1: - resolution: - { - integrity: sha512-XuC6rDl8yOUrctN1nUe5fngPhmq+zhkUvKZVRCUOrlSn5UdLYH0OjrLJl7h8DQdxcAzZw0Oiw+PQFNOBuL42hg==, - } - engines: { node: ">=16", pnpm: ">=7" } - axios@1.6.8: resolution: { @@ -1659,12 +1649,6 @@ packages: engines: { node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7 } hasBin: true - build-url-ts@6.1.7: - resolution: - { - integrity: sha512-KU8hfHxEpBTvoJA4XLiL88JiJN6U0SIVtnXKMSKgxf2T+bP3tDlzd/WIl/dthi+01YXeS49yUxkL3JXyqPdIAw==, - } - callsites@3.1.0: resolution: { @@ -2133,12 +2117,6 @@ packages: integrity: sha512-/sXbVCWayk6GDVg3ctOX6nxaVj7So40FcFAnWlWGNAB1LpYKcV5Cd10APjPjW80O7zYW2MsjBV4zZ7IZO5fVow==, } - http-status-codes@2.3.0: - resolution: - { - integrity: sha512-RJ8XvFvpPM/Dmc5SV+dC4y5PCeOhT3x1Hq0NU3rjGeg5a/CqlhZ7uudknPwZFz4aeAXDcbAyaeP7GAo9lvngtA==, - } - https-proxy-agent@5.0.1: resolution: { @@ -2913,12 +2891,6 @@ packages: integrity: sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==, } - querystringify@2.2.0: - resolution: - { - integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==, - } - react-dom@18.3.1: resolution: { @@ -3080,12 +3052,6 @@ packages: integrity: sha512-z3tJrAs2kIs1AqIIy6pzHmAHlF1hWQ+OdY4/hv+Wxe35EhyLKcajL33iUEn3ScxtFox9nUvRufR/Zre8Q08H/g==, } - requires-port@1.0.0: - resolution: - { - integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==, - } - reselect@4.1.8: resolution: { @@ -3390,12 +3356,6 @@ packages: peerDependencies: browserslist: ">= 4.21.0" - url-parse@1.5.10: - resolution: - { - integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==, - } - vfile-message@4.0.2: resolution: { @@ -4455,16 +4415,6 @@ snapshots: asynckit@0.4.0: {} - axios-tauri-api-adapter@0.2.1: - dependencies: - "@tauri-apps/api": 1.5.4 - axios: 1.6.8 - build-url-ts: 6.1.7 - http-status-codes: 2.3.0 - url-parse: 1.5.10 - transitivePeerDependencies: - - debug - axios@1.6.8: dependencies: follow-redirects: 1.15.6 @@ -4496,8 +4446,6 @@ snapshots: node-releases: 2.0.14 update-browserslist-db: 1.0.15(browserslist@4.23.0) - build-url-ts@6.1.7: {} - callsites@3.1.0: {} camelcase@6.3.0: {} @@ -4774,8 +4722,6 @@ snapshots: html-url-attributes@3.0.0: {} - http-status-codes@2.3.0: {} - https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 @@ -5293,8 +5239,6 @@ snapshots: end-of-stream: 1.4.4 once: 1.4.0 - querystringify@2.2.0: {} - react-dom@18.3.1(react@18.3.1): dependencies: loose-envify: 1.4.0 @@ -5404,8 +5348,6 @@ snapshots: unified: 11.0.4 vfile: 6.0.1 - requires-port@1.0.0: {} - reselect@4.1.8: {} resize-observer-polyfill@1.5.1: {} @@ -5581,11 +5523,6 @@ snapshots: escalade: 3.1.2 picocolors: 1.0.0 - url-parse@1.5.10: - dependencies: - querystringify: 2.2.0 - requires-port: 1.0.0 - vfile-message@4.0.2: dependencies: "@types/unist": 3.0.2 diff --git a/clash-verge-rev/src-tauri/Cargo.lock b/clash-verge-rev/src-tauri/Cargo.lock index 70c55c45e5..50433c12ab 100644 --- a/clash-verge-rev/src-tauri/Cargo.lock +++ b/clash-verge-rev/src-tauri/Cargo.lock @@ -788,7 +788,7 @@ dependencies = [ [[package]] name = "clash-verge" -version = "1.6.2" +version = "1.6.4" dependencies = [ "anyhow", "auto-launch", @@ -5169,7 +5169,7 @@ dependencies = [ [[package]] name = "sysproxy" version = "0.3.0" -source = "git+https://github.com/zzzgydi/sysproxy-rs?branch=main#1402eba27022a2da7b91b3090d2f4936b6260f10" +source = "git+https://github.com/zzzgydi/sysproxy-rs?branch=main#24e8d46cb338a6a8e28742dea6ed993cd6450780" dependencies = [ "interfaces", "iptools", diff --git a/clash-verge-rev/src-tauri/Cargo.toml b/clash-verge-rev/src-tauri/Cargo.toml index b5612a95cc..7a9165026b 100644 --- a/clash-verge-rev/src-tauri/Cargo.toml +++ b/clash-verge-rev/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clash-verge" -version = "1.6.2" +version = "1.6.4" description = "clash verge" authors = ["zzzgydi", "wonfen", "MystiPanda"] license = "GPL-3.0-only" @@ -37,7 +37,7 @@ serde = { version = "1.0", features = ["derive"] } reqwest = { version = "0.12", features = ["json", "rustls-tls"] } sysproxy = { git="https://github.com/zzzgydi/sysproxy-rs", branch = "main" } auto-launch = { git="https://github.com/zzzgydi/auto-launch", branch = "main" } -tauri = { version = "1.6", features = [ "http-all", "fs-read-file", "fs-exists", "path-all", "protocol-asset", "dialog-open", "notification-all", "icon-png", "icon-ico", "clipboard-all", "global-shortcut-all", "process-all", "shell-all", "system-tray", "updater", "window-all", "devtools"] } +tauri = { version = "1.6", features = [ "fs-read-file", "fs-exists", "path-all", "protocol-asset", "dialog-open", "notification-all", "icon-png", "icon-ico", "clipboard-all", "global-shortcut-all", "process-all", "shell-all", "system-tray", "updater", "window-all", "devtools"] } [target.'cfg(windows)'.dependencies] runas = "=1.2.0" diff --git a/clash-verge-rev/src-tauri/src/cmds.rs b/clash-verge-rev/src-tauri/src/cmds.rs index 49cc9009d3..c0e54ce623 100644 --- a/clash-verge-rev/src-tauri/src/cmds.rs +++ b/clash-verge-rev/src-tauri/src/cmds.rs @@ -8,7 +8,7 @@ use crate::{ret_err, wrap_err}; use anyhow::{Context, Result}; use serde_yaml::Mapping; use std::collections::{HashMap, VecDeque}; -use sysproxy::Sysproxy; +use sysproxy::{Autoproxy, Sysproxy}; use tauri::{api, Manager}; type CmdResult = Result; @@ -194,7 +194,6 @@ pub fn grant_permission(_core: String) -> CmdResult { #[tauri::command] pub fn get_sys_proxy() -> CmdResult { let current = wrap_err!(Sysproxy::get_system_proxy())?; - let mut map = Mapping::new(); map.insert("enable".into(), current.enable.into()); map.insert( @@ -206,6 +205,18 @@ pub fn get_sys_proxy() -> CmdResult { Ok(map) } +/// get the system proxy +#[tauri::command] +pub fn get_auto_proxy() -> CmdResult { + let current = wrap_err!(Autoproxy::get_auto_proxy())?; + + let mut map = Mapping::new(); + map.insert("enable".into(), current.enable.into()); + map.insert("url".into(), current.url.into()); + + Ok(map) +} + #[tauri::command] pub fn get_clash_logs() -> CmdResult> { Ok(logger::Logger::global().get_log()) diff --git a/clash-verge-rev/src-tauri/src/config/mod.rs b/clash-verge-rev/src-tauri/src/config/mod.rs index b246a76045..8ceaad6d0b 100644 --- a/clash-verge-rev/src-tauri/src/config/mod.rs +++ b/clash-verge-rev/src-tauri/src/config/mod.rs @@ -13,3 +13,8 @@ pub use self::prfitem::*; pub use self::profiles::*; pub use self::runtime::*; pub use self::verge::*; + +pub const DEFAULT_PAC: &str = r#"function FindProxyForURL(url, host) { + return "PROXY 127.0.0.1:%mixed-port%; SOCKS5 127.0.0.1:%mixed-port%; DIRECT;"; +} +"#; diff --git a/clash-verge-rev/src-tauri/src/config/verge.rs b/clash-verge-rev/src-tauri/src/config/verge.rs index fe1f21e5ef..940b36026c 100644 --- a/clash-verge-rev/src-tauri/src/config/verge.rs +++ b/clash-verge-rev/src-tauri/src/config/verge.rs @@ -1,3 +1,4 @@ +use crate::config::DEFAULT_PAC; use crate::utils::{dirs, help}; use anyhow::Result; use log::LevelFilter; @@ -80,6 +81,12 @@ pub struct IVerge { /// proxy guard duration pub proxy_guard_duration: Option, + /// use pac mode + pub proxy_auto_config: Option, + + /// pac script content + pub pac_file_content: Option, + /// theme setting pub theme_setting: Option, @@ -211,6 +218,8 @@ impl IVerge { enable_auto_launch: Some(false), enable_silent_start: Some(false), enable_system_proxy: Some(false), + proxy_auto_config: Some(false), + pac_file_content: Some(DEFAULT_PAC.into()), enable_random_port: Some(false), #[cfg(not(target_os = "windows"))] verge_redir_port: Some(7895), @@ -290,6 +299,8 @@ impl IVerge { patch!(enable_proxy_guard); patch!(system_proxy_bypass); patch!(proxy_guard_duration); + patch!(proxy_auto_config); + patch!(pac_file_content); patch!(theme_setting); patch!(web_ui_list); diff --git a/clash-verge-rev/src-tauri/src/core/sysopt.rs b/clash-verge-rev/src-tauri/src/core/sysopt.rs index 23b608bca5..2018e54cb0 100644 --- a/clash-verge-rev/src-tauri/src/core/sysopt.rs +++ b/clash-verge-rev/src-tauri/src/core/sysopt.rs @@ -1,11 +1,14 @@ -use crate::{config::Config, log_err}; +use crate::{ + config::{Config, IVerge}, + log_err, +}; use anyhow::{anyhow, Result}; use auto_launch::{AutoLaunch, AutoLaunchBuilder}; use once_cell::sync::OnceCell; use parking_lot::Mutex; use std::env::current_exe; use std::sync::Arc; -use sysproxy::Sysproxy; +use sysproxy::{Autoproxy, Sysproxy}; use tauri::async_runtime::Mutex as TokioMutex; pub struct Sysopt { @@ -16,6 +19,13 @@ pub struct Sysopt { /// recover it when exit old_sysproxy: Arc>>, + /// current auto proxy setting + cur_autoproxy: Arc>>, + + /// record the original auto proxy + /// recover it when exit + old_autoproxy: Arc>>, + /// helps to auto launch the app auto_launch: Arc>>, @@ -38,6 +48,8 @@ impl Sysopt { SYSOPT.get_or_init(|| Sysopt { cur_sysproxy: Arc::new(Mutex::new(None)), old_sysproxy: Arc::new(Mutex::new(None)), + cur_autoproxy: Arc::new(Mutex::new(None)), + old_autoproxy: Arc::new(Mutex::new(None)), auto_launch: Arc::new(Mutex::new(None)), guard_state: Arc::new(TokioMutex::new(false)), }) @@ -49,17 +61,18 @@ impl Sysopt { .latest() .verge_mixed_port .unwrap_or(Config::clash().data().get_mixed_port()); + let pac_port = IVerge::get_singleton_port(); - let (enable, bypass) = { + let (enable, bypass, pac) = { let verge = Config::verge(); let verge = verge.latest(); ( verge.enable_system_proxy.unwrap_or(false), verge.system_proxy_bypass.clone(), + verge.proxy_auto_config.unwrap_or(false), ) }; - - let current = Sysproxy { + let mut sys = Sysproxy { enable, host: String::from("127.0.0.1"), port, @@ -74,13 +87,32 @@ impl Sysopt { None => DEFAULT_BYPASS.into(), }, }; - - if enable { + let mut auto = Autoproxy { + enable, + url: format!("http://127.0.0.1:{pac_port}/commands/pac"), + }; + if pac { + sys.enable = false; let old = Sysproxy::get_system_proxy().ok(); - current.set_system_proxy()?; - + sys.set_system_proxy()?; *self.old_sysproxy.lock() = old; - *self.cur_sysproxy.lock() = Some(current); + *self.cur_sysproxy.lock() = Some(sys); + + let old = Autoproxy::get_auto_proxy().ok(); + auto.set_auto_proxy()?; + *self.old_autoproxy.lock() = old; + *self.cur_autoproxy.lock() = Some(auto); + } else { + auto.enable = false; + let old = Autoproxy::get_auto_proxy().ok(); + auto.set_auto_proxy()?; + *self.old_autoproxy.lock() = old; + *self.cur_autoproxy.lock() = Some(auto); + + let old = Sysproxy::get_system_proxy().ok(); + sys.set_system_proxy()?; + *self.old_sysproxy.lock() = old; + *self.cur_sysproxy.lock() = Some(sys); } // run the system proxy guard @@ -92,24 +124,38 @@ impl Sysopt { pub fn update_sysproxy(&self) -> Result<()> { let mut cur_sysproxy = self.cur_sysproxy.lock(); let old_sysproxy = self.old_sysproxy.lock(); + let mut cur_autoproxy = self.cur_autoproxy.lock(); + let old_autoproxy = self.old_autoproxy.lock(); - if cur_sysproxy.is_none() || old_sysproxy.is_none() { - drop(cur_sysproxy); - drop(old_sysproxy); - return self.init_sysproxy(); - } - - let (enable, bypass) = { + let (enable, bypass, pac) = { let verge = Config::verge(); let verge = verge.latest(); ( verge.enable_system_proxy.unwrap_or(false), verge.system_proxy_bypass.clone(), + verge.proxy_auto_config.unwrap_or(false), ) }; - let mut sysproxy = cur_sysproxy.take().unwrap(); + if pac { + if cur_autoproxy.is_none() || old_autoproxy.is_none() { + drop(cur_autoproxy); + drop(old_autoproxy); + return self.init_sysproxy(); + } + } else { + if cur_sysproxy.is_none() || old_sysproxy.is_none() { + drop(cur_sysproxy); + drop(old_sysproxy); + return self.init_sysproxy(); + } + } + let port = Config::verge() + .latest() + .verge_mixed_port + .unwrap_or(Config::clash().data().get_mixed_port()); + let pac_port = IVerge::get_singleton_port(); - sysproxy.enable = enable; + let mut sysproxy = cur_sysproxy.take().unwrap(); sysproxy.bypass = match bypass { Some(bypass) => { if bypass.is_empty() { @@ -120,15 +166,26 @@ impl Sysopt { } None => DEFAULT_BYPASS.into(), }; - - let port = Config::verge() - .latest() - .verge_mixed_port - .unwrap_or(Config::clash().data().get_mixed_port()); sysproxy.port = port; - sysproxy.set_system_proxy()?; - *cur_sysproxy = Some(sysproxy); + let mut autoproxy = cur_autoproxy.take().unwrap(); + autoproxy.url = format!("http://127.0.0.1:{pac_port}/commands/pac"); + + if pac { + sysproxy.enable = false; + sysproxy.set_system_proxy()?; + *cur_sysproxy = Some(sysproxy); + autoproxy.enable = enable; + autoproxy.set_auto_proxy()?; + *cur_autoproxy = Some(autoproxy); + } else { + autoproxy.enable = false; + autoproxy.set_auto_proxy()?; + *cur_autoproxy = Some(autoproxy); + sysproxy.enable = enable; + sysproxy.set_system_proxy()?; + *cur_sysproxy = Some(sysproxy); + } Ok(()) } @@ -137,8 +194,11 @@ impl Sysopt { pub fn reset_sysproxy(&self) -> Result<()> { let mut cur_sysproxy = self.cur_sysproxy.lock(); let mut old_sysproxy = self.old_sysproxy.lock(); + let mut cur_autoproxy = self.cur_autoproxy.lock(); + let mut old_autoproxy = self.old_autoproxy.lock(); let cur_sysproxy = cur_sysproxy.take(); + let cur_autoproxy = cur_autoproxy.take(); if let Some(mut old) = old_sysproxy.take() { // 如果原代理和当前代理 端口一致，就disable关闭，否则就恢复原代理设置 @@ -162,6 +222,28 @@ impl Sysopt { log::info!(target: "app", "reset proxy with no action"); } + if let Some(mut old) = old_autoproxy.take() { + // 如果原代理和当前代理 URL一致，就disable关闭，否则就恢复原代理设置 + // 当前没有设置代理的时候，不确定旧设置是否和当前一致，全关了 + let url_same = cur_autoproxy.map_or(true, |cur| old.url == cur.url); + + if old.enable && url_same { + old.enable = false; + log::info!(target: "app", "reset proxy by disabling the original proxy"); + } else { + log::info!(target: "app", "reset proxy to the original proxy"); + } + + old.set_auto_proxy()?; + } else if let Some(mut cur @ Autoproxy { enable: true, .. }) = cur_autoproxy { + // 没有原代理，就按现在的代理设置disable即可 + log::info!(target: "app", "reset proxy by disabling the current proxy"); + cur.enable = false; + cur.set_auto_proxy()?; + } else { + log::info!(target: "app", "reset proxy with no action"); + } + Ok(()) } @@ -267,7 +349,7 @@ impl Sysopt { loop { sleep(Duration::from_secs(wait_secs)).await; - let (enable, guard, guard_duration, bypass) = { + let (enable, guard, guard_duration, bypass, pac) = { let verge = Config::verge(); let verge = verge.latest(); ( @@ -275,6 +357,7 @@ impl Sysopt { verge.enable_proxy_guard.unwrap_or(false), verge.proxy_guard_duration.unwrap_or(10), verge.system_proxy_bypass.clone(), + verge.proxy_auto_config.unwrap_or(false), ) }; @@ -294,24 +377,32 @@ impl Sysopt { .verge_mixed_port .unwrap_or(Config::clash().data().get_mixed_port()) }; - - let sysproxy = Sysproxy { - enable: true, - host: "127.0.0.1".into(), - port, - bypass: match bypass { - Some(bypass) => { - if bypass.is_empty() { - DEFAULT_BYPASS.into() - } else { - bypass + let pac_port = IVerge::get_singleton_port(); + if pac { + let autoproxy = Autoproxy { + enable: true, + url: format!("http://127.0.0.1:{pac_port}/commands/pac"), + }; + log_err!(autoproxy.set_auto_proxy()); + } else { + let sysproxy = Sysproxy { + enable: true, + host: "127.0.0.1".into(), + port, + bypass: match bypass { + Some(bypass) => { + if bypass.is_empty() { + DEFAULT_BYPASS.into() + } else { + bypass + } } - } - None => DEFAULT_BYPASS.into(), - }, - }; + None => DEFAULT_BYPASS.into(), + }, + }; - log_err!(sysproxy.set_system_proxy()); + log_err!(sysproxy.set_system_proxy()); + } } let mut state = guard_state.lock().await; diff --git a/clash-verge-rev/src-tauri/src/feat.rs b/clash-verge-rev/src-tauri/src/feat.rs index bed5b8a0e3..f590351615 100644 --- a/clash-verge-rev/src-tauri/src/feat.rs +++ b/clash-verge-rev/src-tauri/src/feat.rs @@ -177,6 +177,8 @@ pub async fn patch_verge(patch: IVerge) -> Result<()> { let tun_mode = patch.enable_tun_mode; let auto_launch = patch.enable_auto_launch; let system_proxy = patch.enable_system_proxy; + let pac = patch.proxy_auto_config; + let pac_content = patch.pac_file_content; let proxy_bypass = patch.system_proxy_bypass; let language = patch.language; let port = patch.verge_mixed_port; @@ -219,7 +221,12 @@ pub async fn patch_verge(patch: IVerge) -> Result<()> { if auto_launch.is_some() { sysopt::Sysopt::global().update_launch()?; } - if system_proxy.is_some() || proxy_bypass.is_some() || port.is_some() { + if system_proxy.is_some() + || proxy_bypass.is_some() + || port.is_some() + || pac.is_some() + || pac_content.is_some() + { sysopt::Sysopt::global().update_sysproxy()?; sysopt::Sysopt::global().guard_proxy(); } diff --git a/clash-verge-rev/src-tauri/src/main.rs b/clash-verge-rev/src-tauri/src/main.rs index 2eeb52abec..738135c60d 100644 --- a/clash-verge-rev/src-tauri/src/main.rs +++ b/clash-verge-rev/src-tauri/src/main.rs @@ -36,6 +36,7 @@ fn main() -> std::io::Result<()> { .invoke_handler(tauri::generate_handler![ // common cmds::get_sys_proxy, + cmds::get_auto_proxy, cmds::open_app_dir, cmds::open_logs_dir, cmds::open_web_url, diff --git a/clash-verge-rev/src-tauri/src/utils/server.rs b/clash-verge-rev/src-tauri/src/utils/server.rs index cd1fb906df..8bad353321 100644 --- a/clash-verge-rev/src-tauri/src/utils/server.rs +++ b/clash-verge-rev/src-tauri/src/utils/server.rs @@ -1,7 +1,7 @@ extern crate warp; use super::resolve; -use crate::config::IVerge; +use crate::config::{Config, IVerge, DEFAULT_PAC}; use anyhow::{bail, Result}; use port_scanner::local_port_available; use std::convert::Infallible; @@ -64,6 +64,22 @@ pub fn embed_server(app_handle: AppHandle) { "ok" }); + let pac = warp::path!("commands" / "pac").map(move || { + let content = Config::verge() + .latest() + .pac_file_content + .clone() + .unwrap_or(DEFAULT_PAC.to_string()); + let port = Config::verge() + .latest() + .verge_mixed_port + .unwrap_or(Config::clash().data().get_mixed_port()); + let content = content.replace("%mixed-port%", &format!("{}", port)); + warp::http::Response::builder() + .header("Content-Type", "application/x-ns-proxy-autoconfig") + .body(content) + .unwrap_or_default() + }); let scheme = warp::path!("commands" / "scheme") .and(warp::query::()) .and_then(scheme_handler); @@ -72,7 +88,7 @@ pub fn embed_server(app_handle: AppHandle) { resolve::resolve_scheme(query.param).await; Ok("ok") } - let commands = ping.or(visible).or(scheme); + let commands = ping.or(visible).or(pac).or(scheme); warp::serve(commands).run(([127, 0, 0, 1], port)).await; }); } diff --git a/clash-verge-rev/src-tauri/tauri.conf.json b/clash-verge-rev/src-tauri/tauri.conf.json index 7a03aba89a..3b909b7b47 100644 --- a/clash-verge-rev/src-tauri/tauri.conf.json +++ b/clash-verge-rev/src-tauri/tauri.conf.json @@ -2,7 +2,7 @@ "$schema": "../node_modules/@tauri-apps/cli/schema.json", "package": { "productName": "Clash Verge", - "version": "1.6.2" + "version": "1.6.4" }, "build": { "distDir": "../dist", @@ -38,10 +38,6 @@ "pubkey": "dW50cnVzdGVkIGNvbW1lbnQ6IG1pbmlzaWduIHB1YmxpYyBrZXk6IEQyOEMyRjBCQkVGOUJEREYKUldUZnZmbStDeStNMHU5Mmo1N24xQXZwSVRYbXA2NUpzZE5oVzlqeS9Bc0t6RVV4MmtwVjBZaHgK" }, "allowlist": { - "http": { - "all": true, - "scope": ["http://**", "http://**"] - }, "shell": { "all": true }, diff --git a/clash-verge-rev/src/components/layout/layout-control.tsx b/clash-verge-rev/src/components/layout/layout-control.tsx index ad72693bee..6a69051ad5 100644 --- a/clash-verge-rev/src/components/layout/layout-control.tsx +++ b/clash-verge-rev/src/components/layout/layout-control.tsx @@ -36,6 +36,7 @@ export const LayoutControl = () => { ({ + suggestions: [ + { + label: "%mixed-port%", + kind: monaco.languages.CompletionItemKind.Text, + insertText: "%mixed-port%", + range: { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: model.getWordUntilPosition(position).startColumn - 1, + endColumn: model.getWordUntilPosition(position).endColumn - 1, + }, + }, + ], + }), +}); export const EditorViewer = (props: Props) => { const { @@ -114,7 +131,7 @@ export const EditorViewer = (props: Props) => { padding: { top: 33, // 顶部padding防止遮挡snippets }, - fontFamily: `Fira Code, Roboto Mono, Roboto, Source Code Pro, Menlo, Monaco, Consolas, Courier New, monospace, "Apple Color Emoji"${ + fontFamily: `Fira Code, JetBrains Mono, Roboto Mono, "Source Code Pro", Consolas, Menlo, Monaco, monospace, "Courier New", "Apple Color Emoji"${ getSystem() === "windows" ? ", twemoji mozilla" : "" }`, fontLigatures: true, // 连字符 diff --git a/clash-verge-rev/src/components/setting/mods/sysproxy-viewer.tsx b/clash-verge-rev/src/components/setting/mods/sysproxy-viewer.tsx index 8f2d94d670..19e2de3490 100644 --- a/clash-verge-rev/src/components/setting/mods/sysproxy-viewer.tsx +++ b/clash-verge-rev/src/components/setting/mods/sysproxy-viewer.tsx @@ -10,23 +10,34 @@ import { styled, TextField, Typography, + Button, } from "@mui/material"; import { useVerge } from "@/hooks/use-verge"; -import { getSystemProxy } from "@/services/cmds"; +import { getSystemProxy, getAutotemProxy } from "@/services/cmds"; import { BaseDialog, DialogRef, Notice, Switch } from "@/components/base"; +import { Edit } from "@mui/icons-material"; +import { EditorViewer } from "@/components/profile/editor-viewer"; +const DEFAULT_PAC = `function FindProxyForURL(url, host) { + return "PROXY 127.0.0.1:%mixed-port%; SOCKS5 127.0.0.1:%mixed-port%; DIRECT;"; +}`; export const SysproxyViewer = forwardRef((props, ref) => { const { t } = useTranslation(); const [open, setOpen] = useState(false); - + const [editorOpen, setEditorOpen] = useState(false); const { verge, patchVerge } = useVerge(); type SysProxy = Awaited>; const [sysproxy, setSysproxy] = useState(); + type AutoProxy = Awaited>; + const [autoproxy, setAutoproxy] = useState(); + const { enable_system_proxy: enabled, + proxy_auto_config, + pac_file_content, enable_proxy_guard, system_proxy_bypass, proxy_guard_duration, @@ -36,6 +47,8 @@ export const SysproxyViewer = forwardRef((props, ref) => { guard: enable_proxy_guard, bypass: system_proxy_bypass, duration: proxy_guard_duration ?? 10, + pac: proxy_auto_config, + pac_content: pac_file_content ?? DEFAULT_PAC, }); useImperativeHandle(ref, () => ({ @@ -45,8 +58,11 @@ export const SysproxyViewer = forwardRef((props, ref) => { guard: enable_proxy_guard, bypass: system_proxy_bypass, duration: proxy_guard_duration ?? 10, + pac: proxy_auto_config, + pac_content: pac_file_content ?? DEFAULT_PAC, }); getSystemProxy().then((p) => setSysproxy(p)); + getAutotemProxy().then((p) => setAutoproxy(p)); }, close: () => setOpen(false), })); @@ -68,6 +84,12 @@ export const SysproxyViewer = forwardRef((props, ref) => { if (value.bypass !== system_proxy_bypass) { patch.system_proxy_bypass = value.bypass; } + if (value.pac !== proxy_auto_config) { + patch.proxy_auto_config = value.pac; + } + if (value.pac_content !== pac_file_content) { + patch.pac_file_content = value.pac_content; + } try { await patchVerge(patch); @@ -89,6 +111,15 @@ export const SysproxyViewer = forwardRef((props, ref) => { onOk={onSave} > + + + setValue((v) => ({ ...v, pac: e }))} + /> + ((props, ref) => { }} /> - - - - - - - setValue((v) => ({ ...v, bypass: e.target.value })) - } - /> - + {!value.pac && ( + <> + + + + + + setValue((v) => ({ ...v, bypass: e.target.value })) + } + /> + + + )} + {value.pac && ( + <> + + + + { + let pac = DEFAULT_PAC; + if (content && content.trim().length > 0) { + pac = content; + } + setValue((v) => ({ ...v, pac_content: pac })); + }} + onClose={() => { + setEditorOpen(false); + }} + /> + + + )} @@ -146,29 +219,42 @@ export const SysproxyViewer = forwardRef((props, ref) => { {t("Enable status")} - {(!!sysproxy?.enable).toString()} + {value.pac + ? (!!autoproxy?.enable).toString() + : (!!sysproxy?.enable).toString()} + {!value.pac && ( + <> + + {t("Server Addr")} + + {sysproxy?.server || "-"} + + - - {t("Server Addr")} - {sysproxy?.server || "-"} - - - - {t("Bypass")} - - - - + + {t("Bypass")} + + + + + + )} + {value.pac && ( + + {t("PAC URL")} + {autoproxy?.url || "-"} + + )} ); diff --git a/clash-verge-rev/src/locales/en.json b/clash-verge-rev/src/locales/en.json index 0aa249d63b..d22672ca8f 100644 --- a/clash-verge-rev/src/locales/en.json +++ b/clash-verge-rev/src/locales/en.json @@ -118,6 +118,9 @@ "System Proxy Setting": "System Proxy Setting", "Open UWP tool": "Open UWP tool", "Update GeoData": "Update GeoData", + "Use PAC Mode": "Use PAC Mode", + "PAC URL": "PAC URL", + "PAC Script Content": "PAC Script Content", "Proxy Guard": "Proxy Guard", "Guard Duration": "Guard Duration", "Proxy Bypass": "Proxy Bypass", diff --git a/clash-verge-rev/src/locales/fa.json b/clash-verge-rev/src/locales/fa.json index bba17b33d4..11ed46275a 100644 --- a/clash-verge-rev/src/locales/fa.json +++ b/clash-verge-rev/src/locales/fa.json @@ -118,6 +118,9 @@ "System Proxy Setting": "تنظیمات پراکسی سیستم", "Open UWP tool": "باز کردن ابزار UWP", "Update GeoData": "به‌روزرسانی GeoData", + "Use PAC Mode": "استفاده از حالت PAC", + "PAC URL": "PAC URL", + "PAC Script Content": "محتوای اسکریپت PAC", "Proxy Guard": "محافظ پراکسی", "Guard Duration": "مدت محافظت", "Proxy Bypass": "دور زدن پراکسی", diff --git a/clash-verge-rev/src/locales/ru.json b/clash-verge-rev/src/locales/ru.json index e4d7636acc..0047d5c5c4 100644 --- a/clash-verge-rev/src/locales/ru.json +++ b/clash-verge-rev/src/locales/ru.json @@ -118,6 +118,9 @@ "System Proxy Setting": "Настройка системного прокси", "Open UWP tool": "Открыть UWP инструмент", "Update GeoData": "Обновление GeoData", + "Use PAC Mode": "Используйте режим PAC.", + "PAC URL": "Адрес PAC", + "PAC Script Content": "Содержание сценария PAC", "Proxy Guard": "Защита прокси", "Guard Duration": "Период защиты", "Proxy Bypass": "Игнорирование прокси", diff --git a/clash-verge-rev/src/locales/zh.json b/clash-verge-rev/src/locales/zh.json index 22cec44cb1..dc7ed0b679 100644 --- a/clash-verge-rev/src/locales/zh.json +++ b/clash-verge-rev/src/locales/zh.json @@ -118,6 +118,9 @@ "System Proxy Setting": "系统代理设置", "Open UWP tool": "UWP 工具", "Update GeoData": "更新 GeoData", + "Use PAC Mode": "使用PAC模式", + "PAC URL": "PAC 地址", + "PAC Script Content": "PAC 脚本内容", "Proxy Guard": "系统代理守卫", "Guard Duration": "代理守卫间隔", "Proxy Bypass": "代理绕过", diff --git a/clash-verge-rev/src/services/api.ts b/clash-verge-rev/src/services/api.ts index 42e6bcd8fd..bded6e32e1 100644 --- a/clash-verge-rev/src/services/api.ts +++ b/clash-verge-rev/src/services/api.ts @@ -1,4 +1,3 @@ -import axiosTauriApiAdapter from "axios-tauri-api-adapter"; import axios, { AxiosInstance } from "axios"; import { getClashInfo } from "./cmds"; @@ -26,7 +25,6 @@ export const getAxios = async (force: boolean = false) => { } catch {} axiosIns = axios.create({ - adapter: axiosTauriApiAdapter, baseURL: `http://${server}`, headers: secret ? { Authorization: `Bearer ${secret}` } : {}, timeout: 15000, diff --git a/clash-verge-rev/src/services/cmds.ts b/clash-verge-rev/src/services/cmds.ts index 9b7ac1b9cc..b91ccec998 100644 --- a/clash-verge-rev/src/services/cmds.ts +++ b/clash-verge-rev/src/services/cmds.ts @@ -127,6 +127,13 @@ export async function getSystemProxy() { }>("get_sys_proxy"); } +export async function getAutotemProxy() { + return invoke<{ + enable: boolean; + url: string; + }>("get_auto_proxy"); +} + export async function changeClashCore(clashCore: string) { return invoke("change_clash_core", { clashCore }); } diff --git a/clash-verge-rev/src/services/types.d.ts b/clash-verge-rev/src/services/types.d.ts index 4ede695f67..13734526fd 100644 --- a/clash-verge-rev/src/services/types.d.ts +++ b/clash-verge-rev/src/services/types.d.ts @@ -216,6 +216,8 @@ interface IVergeConfig { enable_service_mode?: boolean; enable_silent_start?: boolean; enable_system_proxy?: boolean; + proxy_auto_config?: boolean; + pac_file_content?: string; enable_random_port?: boolean; verge_mixed_port?: number; verge_socks_port?: number; diff --git a/echo/internal/transporter/base.go b/echo/internal/transporter/base.go index e9e7e47206..6ebca196da 100644 --- a/echo/internal/transporter/base.go +++ b/echo/internal/transporter/base.go @@ -61,7 +61,8 @@ func (b *baseTransporter) RelayTCPConn(c net.Conn, handshakeF TCPHandShakeF) err ctx, c, buffer, constant.SniffTimeOut, sniff.TLSClientHello, sniff.HTTPHost) if err != nil { - b.l.Warnf("sniff error: %s", err) + // this mean no protocol sniffed + b.l.Debug("sniff error: %s", err) } if sniffMetadata != nil { b.l.Infof("sniffed protocol: %s", sniffMetadata.Protocol) diff --git a/lede/include/kernel-5.10 b/lede/include/kernel-5.10 index 23d4abf667..3355c82098 100644 --- a/lede/include/kernel-5.10 +++ b/lede/include/kernel-5.10 @@ -1,2 +1,2 @@ -LINUX_VERSION-5.10 = .217 -LINUX_KERNEL_HASH-5.10.217 = c52bc1ffc396c11bce335c9ee5cd55fe4213cbc1fb4026ff62bb90c864c61f62 +LINUX_VERSION-5.10 = .218 +LINUX_KERNEL_HASH-5.10.218 = 9c36b243e8c3ec1d5963366618f336710b84340bf95be2037b26c452392cb2d6 diff --git a/lede/include/kernel-5.15 b/lede/include/kernel-5.15 index 7fd86427d3..39c74ab2ae 100644 --- a/lede/include/kernel-5.15 +++ b/lede/include/kernel-5.15 @@ -1,2 +1,2 @@ -LINUX_VERSION-5.15 = .159 -LINUX_KERNEL_HASH-5.15.159 = 3478fe50225e9c88e09cf114f38e3fb71e82d9fdcc356aa1257d721199b341c5 +LINUX_VERSION-5.15 = .160 +LINUX_KERNEL_HASH-5.15.160 = f41e718e33b88f269a6b6a7653e5e9824c4ba541f6ffe5bf26ecc37c540a1b05 diff --git a/lede/include/kernel-5.4 b/lede/include/kernel-5.4 index eb8d7dd967..bb474605fd 100644 --- a/lede/include/kernel-5.4 +++ b/lede/include/kernel-5.4 @@ -1,2 +1,2 @@ -LINUX_VERSION-5.4 = .276 -LINUX_KERNEL_HASH-5.4.276 = e2712ebd4421ffa5b25a366659ecfbe1b45a444027ee2fe57369676453e86e07 +LINUX_VERSION-5.4 = .277 +LINUX_KERNEL_HASH-5.4.277 = 7e1f5b28588e49ddfd18e7772476e4e8b52bdc9c3e19beafcbb7c103e6c01f51 diff --git a/lede/include/kernel-6.1 b/lede/include/kernel-6.1 index 0b15ce2dc3..026ef91a52 100644 --- a/lede/include/kernel-6.1 +++ b/lede/include/kernel-6.1 @@ -1,2 +1,2 @@ -LINUX_VERSION-6.1 = .91 -LINUX_KERNEL_HASH-6.1.91 = 880ace63ca2291b8b639e9bd862cc828649d3e1e00ccfee5861473debd2e4dec +LINUX_VERSION-6.1 = .92 +LINUX_KERNEL_HASH-6.1.92 = 9019f427bfdc9ced5bc954d760d37ac08c0cdffb45ad28087fc45a73e64336c9 diff --git a/lede/include/kernel-6.6 b/lede/include/kernel-6.6 index 8fd2d83189..896c95ea7d 100644 --- a/lede/include/kernel-6.6 +++ b/lede/include/kernel-6.6 @@ -1,2 +1,2 @@ -LINUX_VERSION-6.6 = .31 -LINUX_KERNEL_HASH-6.6.31 = d6ecff966f8c95ec4cb3bb303904f757b7de6a6bcfef0d0771cb852158e61c20 +LINUX_VERSION-6.6 = .32 +LINUX_KERNEL_HASH-6.6.32 = aaa824eaf07f61911d22b75ff090a403c3dd0bd73e23933e0bba8b5971436ce1 diff --git a/lede/package/base-files/Makefile b/lede/package/base-files/Makefile index 3f6e25d605..b3656cd3e0 100644 --- a/lede/package/base-files/Makefile +++ b/lede/package/base-files/Makefile @@ -23,6 +23,7 @@ PKG_LICENSE:=GPL-2.0 PKG_CONFIG_DEPENDS += \ CONFIG_SIGNED_PACKAGES CONFIG_TARGET_INIT_PATH CONFIG_TARGET_PREINIT_DISABLE_FAILSAFE \ CONFIG_NAND_SUPPORT \ + CONFIG_LEGACY_SDCARD_SUPPORT \ CONFIG_EMMC_SUPPORT \ CONFIG_CLEAN_IPKG \ CONFIG_PER_FEED_REPO \ @@ -125,6 +126,12 @@ ifeq ($(CONFIG_NAND_SUPPORT),) endef endif +ifeq ($(CONFIG_LEGACY_SDCARD_SUPPORT),) + define Package/base-files/legacy-sdcard-support + rm -f $(1)/lib/upgrade/legacy-sdcard.sh + endef +endif + ifeq ($(CONFIG_EMMC_SUPPORT),) define Package/base-files/emmc-support rm -f $(1)/lib/upgrade/emmc.sh @@ -135,6 +142,7 @@ define Package/base-files/install $(CP) ./files/* $(1)/ $(Package/base-files/install-key) $(Package/base-files/nand-support) + $(Package/base-files/legacy-sdcard-support) $(Package/base-files/emmc-support) if [ -d $(GENERIC_PLATFORM_DIR)/base-files/. ]; then \ $(CP) $(GENERIC_PLATFORM_DIR)/base-files/* $(1)/; \ diff --git a/lede/package/base-files/files/lib/upgrade/legacy-sdcard.sh b/lede/package/base-files/files/lib/upgrade/legacy-sdcard.sh new file mode 100644 index 0000000000..d2ae53b729 --- /dev/null +++ b/lede/package/base-files/files/lib/upgrade/legacy-sdcard.sh @@ -0,0 +1,91 @@ +legacy_sdcard_check_image() { + local file="$1" + local diskdev partdev diff + + export_bootdevice && export_partdevice diskdev 0 || { + v "Unable to determine upgrade device" + return 1 + } + + get_partitions "/dev/$diskdev" bootdisk + + v "Extract boot sector from the image" + get_image_dd "$1" of=/tmp/image.bs count=1 bs=512b + + get_partitions /tmp/image.bs image + + #compare tables + diff="$(grep -F -x -v -f /tmp/partmap.bootdisk /tmp/partmap.image)" + + rm -f /tmp/image.bs /tmp/partmap.bootdisk /tmp/partmap.image + + if [ -n "$diff" ]; then + v "Partition layout has changed. Full image will be written." + ask_bool 0 "Abort" && exit 1 + return 0 + fi +} + +legacy_sdcard_do_upgrade() { + local board=$(board_name) + local diskdev partdev diff + + export_bootdevice && export_partdevice diskdev 0 || { + v "Unable to determine upgrade device" + return 1 + } + + sync + + if [ "$UPGRADE_OPT_SAVE_PARTITIONS" = "1" ]; then + get_partitions "/dev/$diskdev" bootdisk + + v "Extract boot sector from the image" + get_image_dd "$1" of=/tmp/image.bs count=1 bs=512b + + get_partitions /tmp/image.bs image + + #compare tables + diff="$(grep -F -x -v -f /tmp/partmap.bootdisk /tmp/partmap.image)" + else + diff=1 + fi + + if [ -n "$diff" ]; then + get_image_dd "$1" of="/dev/$diskdev" bs=4096 conv=fsync + + # Separate removal and addtion is necessary; otherwise, partition 1 + # will be missing if it overlaps with the old partition 2 + partx -d - "/dev/$diskdev" + partx -a - "/dev/$diskdev" + else + v "Writing bootloader to /dev/$diskdev" + get_image_dd "$1" of="$diskdev" bs=512 skip=1 seek=1 count=2048 conv=fsync + #iterate over each partition from the image and write it to the boot disk + while read part start size; do + if export_partdevice partdev $part; then + v "Writing image to /dev/$partdev..." + get_image_dd "$1" of="/dev/$partdev" ibs="512" obs=1M skip="$start" count="$size" conv=fsync + else + v "Unable to find partition $part device, skipped." + fi + done < /tmp/partmap.image + + v "Writing new UUID to /dev/$diskdev..." + get_image_dd "$1" of="/dev/$diskdev" bs=1 skip=440 count=4 seek=440 conv=fsync + fi + + sleep 1 +} + +legacy_sdcard_copy_config() { + local partdev + + if export_partdevice partdev 1; then + mkdir -p /boot + [ -f /boot/kernel.img ] || mount -o rw,noatime /dev/$partdev /boot + cp -af "$UPGRADE_BACKUP" "/boot/$BACKUP_FILE" + sync + umount /boot + fi +} diff --git a/lede/package/network/services/hostapd/patches/993-fix-mbo-module-build.patch b/lede/package/network/services/hostapd/patches/993-fix-mbo-module-build.patch new file mode 100644 index 0000000000..3efc484f8c --- /dev/null +++ b/lede/package/network/services/hostapd/patches/993-fix-mbo-module-build.patch @@ -0,0 +1,10 @@ +--- a/wpa_supplicant/Makefile ++++ b/wpa_supplicant/Makefile +@@ -339,6 +339,7 @@ + + ifdef CONFIG_MBO + CONFIG_WNM=y ++NEED_GAS=y + endif + + ifdef CONFIG_WNM diff --git a/lede/target/linux/generic/backport-5.4/782-net-next-1-of-net-pass-the-dst-buffer-to-of_get_mac_address.patch b/lede/target/linux/generic/backport-5.4/782-net-next-1-of-net-pass-the-dst-buffer-to-of_get_mac_address.patch index aa0c251b13..2c1cd9c839 100644 --- a/lede/target/linux/generic/backport-5.4/782-net-next-1-of-net-pass-the-dst-buffer-to-of_get_mac_address.patch +++ b/lede/target/linux/generic/backport-5.4/782-net-next-1-of-net-pass-the-dst-buffer-to-of_get_mac_address.patch @@ -1833,7 +1833,7 @@ Signed-off-by: David S. Miller } --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c -@@ -3467,10 +3467,11 @@ static int bcmgenet_probe(struct platfor +@@ -3478,10 +3478,11 @@ static int bcmgenet_probe(struct platfor const struct of_device_id *of_id = NULL; struct bcmgenet_priv *priv; struct net_device *dev; @@ -1846,7 +1846,7 @@ Signed-off-by: David S. Miller /* Up to GENET_MAX_MQ_CNT + 1 TX queues and RX queues */ dev = alloc_etherdev_mqs(sizeof(*priv), GENET_MAX_MQ_CNT + 1, -@@ -3497,14 +3498,15 @@ static int bcmgenet_probe(struct platfor +@@ -3508,14 +3509,15 @@ static int bcmgenet_probe(struct platfor } if (dn) { @@ -1865,7 +1865,7 @@ Signed-off-by: David S. Miller } priv->base = devm_platform_ioremap_resource(pdev, 0); -@@ -3517,7 +3519,6 @@ static int bcmgenet_probe(struct platfor +@@ -3529,7 +3531,6 @@ static int bcmgenet_probe(struct platfor SET_NETDEV_DEV(dev, &pdev->dev); dev_set_drvdata(&pdev->dev, dev); diff --git a/lede/target/linux/generic/pending-6.6/990-add-missing-dependency-on-CRYPTO_SIG.patch b/lede/target/linux/generic/pending-6.6/990-add-missing-dependency-on-CRYPTO_SIG.patch index 9b07928a9a..641e218574 100644 --- a/lede/target/linux/generic/pending-6.6/990-add-missing-dependency-on-CRYPTO_SIG.patch +++ b/lede/target/linux/generic/pending-6.6/990-add-missing-dependency-on-CRYPTO_SIG.patch @@ -1,10 +1,6 @@ -diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig -index 59ec726b7c77..3f089abd6fc9 100644 --- a/crypto/asymmetric_keys/Kconfig +++ b/crypto/asymmetric_keys/Kconfig -@@ -13,10 +13,11 @@ if ASYMMETRIC_KEY_TYPE - config ASYMMETRIC_PUBLIC_KEY_SUBTYPE - tristate "Asymmetric public-key crypto algorithm subtype" +@@ -15,6 +15,7 @@ config ASYMMETRIC_PUBLIC_KEY_SUBTYPE select MPILIB select CRYPTO_HASH_INFO select CRYPTO_AKCIPHER @@ -12,5 +8,3 @@ index 59ec726b7c77..3f089abd6fc9 100644 select CRYPTO_HASH help This option provides support for asymmetric public key type handling. - If signature generation and/or verification are to be used, - appropriate hash algorithms (such as SHA-1) must be available. diff --git a/lede/target/linux/mediatek/patches-6.1/901-arm-add-cmdline-override.patch b/lede/target/linux/mediatek/patches-6.1/901-arm-add-cmdline-override.patch index bfca4b6389..f8857bdf72 100644 --- a/lede/target/linux/mediatek/patches-6.1/901-arm-add-cmdline-override.patch +++ b/lede/target/linux/mediatek/patches-6.1/901-arm-add-cmdline-override.patch @@ -37,7 +37,7 @@ * CONFIG_CMDLINE is meant to be a default in case nothing else --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig -@@ -2240,6 +2240,14 @@ config CMDLINE_FORCE +@@ -2239,6 +2239,14 @@ config CMDLINE_FORCE endchoice diff --git a/lede/target/linux/mvebu/cortexa72/base-files/etc/board.d/01_leds b/lede/target/linux/mvebu/cortexa72/base-files/etc/board.d/01_leds index 809a64a60e..d8cca08caa 100644 --- a/lede/target/linux/mvebu/cortexa72/base-files/etc/board.d/01_leds +++ b/lede/target/linux/mvebu/cortexa72/base-files/etc/board.d/01_leds @@ -7,9 +7,11 @@ board_config_update board=$(board_name) case "$board" in +qnap,qhora-321|\ iei,puzzle-m901) ucidef_set_led_netdev "wan" "WAN" "white:network" "eth0" "link" ;; +qnap,qhora-322|\ iei,puzzle-m902) ucidef_set_led_netdev "wan" "WAN" "white:network" "eth2" "link" ;; diff --git a/lede/target/linux/mvebu/cortexa72/base-files/etc/board.d/02_network b/lede/target/linux/mvebu/cortexa72/base-files/etc/board.d/02_network index 6a5861084e..c5704c5a37 100644 --- a/lede/target/linux/mvebu/cortexa72/base-files/etc/board.d/02_network +++ b/lede/target/linux/mvebu/cortexa72/base-files/etc/board.d/02_network @@ -13,9 +13,11 @@ case "$board" in globalscale,mochabin) ucidef_set_interfaces_lan_wan "lan0 lan1 lan2 lan3" "eth0 eth2" ;; +qnap,qhora-321|\ iei,puzzle-m901) ucidef_set_interfaces_lan_wan "eth1 eth2 eth3 eth4 eth5" "eth0" ;; +qnap,qhora-322|\ iei,puzzle-m902) ucidef_set_interfaces_lan_wan "eth0 eth1 eth3 eth4 eth5 eth6 eth7 eth8" "eth2" ;; diff --git a/lede/target/linux/mvebu/cortexa72/base-files/lib/upgrade/emmc-puzzle.sh b/lede/target/linux/mvebu/cortexa72/base-files/lib/upgrade/emmc-puzzle.sh index 5e5c356ed6..5c184e5297 100755 --- a/lede/target/linux/mvebu/cortexa72/base-files/lib/upgrade/emmc-puzzle.sh +++ b/lede/target/linux/mvebu/cortexa72/base-files/lib/upgrade/emmc-puzzle.sh @@ -31,6 +31,8 @@ platform_do_upgrade_emmc() { v "Writing new UUID to /dev/$diskdev..." get_image_dd "$1" of="/dev/$diskdev" bs=1 skip=440 count=4 seek=440 conv=fsync + + mkfs.ext4 -F -L rootfs_data $(find_mmc_part rootfs_data) sleep 1 } diff --git a/lede/target/linux/mvebu/cortexa72/base-files/lib/upgrade/platform.sh b/lede/target/linux/mvebu/cortexa72/base-files/lib/upgrade/platform.sh index dc964a3117..527584f69e 100755 --- a/lede/target/linux/mvebu/cortexa72/base-files/lib/upgrade/platform.sh +++ b/lede/target/linux/mvebu/cortexa72/base-files/lib/upgrade/platform.sh @@ -10,6 +10,8 @@ REQUIRE_IMAGE_METADATA=1 platform_check_image() { case "$(board_name)" in globalscale,mochabin|\ + qnap,qhora-321|\ + qnap,qhora-322|\ iei,puzzle-m901|\ iei,puzzle-m902|\ marvell,armada8040-mcbin-doubleshot|\ @@ -25,6 +27,8 @@ platform_check_image() { platform_do_upgrade() { case "$(board_name)" in + qnap,qhora-321|\ + qnap,qhora-322|\ iei,puzzle-m901|\ iei,puzzle-m902) platform_do_upgrade_emmc "$1" @@ -43,6 +47,8 @@ platform_do_upgrade() { platform_copy_config() { case "$(board_name)" in globalscale,mochabin|\ + qnap,qhora-321|\ + qnap,qhora-322|\ iei,puzzle-m901|\ iei,puzzle-m902|\ marvell,armada8040-mcbin-doubleshot|\ diff --git a/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9131-db-A.dts b/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9131-db-A.dts new file mode 100644 index 0000000000..4f84d036f8 --- /dev/null +++ b/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9131-db-A.dts @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT) +/* + * Copyright (C) 2019 Marvell International Ltd. + * + * Device tree for the CN9131-DB board. + */ + +#include "cn9130.dtsi" + +#include +#include + +/ { + model = "QNAP QHora-321"; + compatible = "qnap,qhora-321", + "marvell,armada-ap807-quad", "marvell,armada-ap807"; + + chosen { + stdout-path = "serial0:115200n8"; + bootargs-append = " root=/dev/mmcblk0p3"; + }; + + aliases { + i2c0 = &cp1_i2c0; + i2c1 = &cp0_i2c0; + ethernet0 = &cp0_eth0; + ethernet1 = &cp0_eth1; + ethernet2 = &cp0_eth2; + ethernet3 = &cp1_eth0; + ethernet4 = &cp1_eth1; + ethernet5 = &cp1_eth2; + gpio1 = &cp0_gpio1; + gpio2 = &cp0_gpio2; + gpio3 = &cp1_gpio1; + gpio4 = &cp1_gpio2; + led-boot = &led_power; + led-failsafe = &led_info; + led-running = &led_power; + led-upgrade = &led_info; + }; + + memory@00000000 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x80000000>; + }; + + gpio_keys { + compatible = "gpio-keys"; + + reset { + label = "Reset"; + linux,code = ; + gpios = <&cp0_gpio2 4 GPIO_ACTIVE_LOW>; + }; + }; +}; + +&uart0 { + status = "okay"; +}; + +&cp0_uart0 { + status = "okay"; + + puzzle-mcu { + compatible = "iei,wt61p803-puzzle"; + #address-cells = <1>; + #size-cells = <1>; + current-speed = <115200>; + enable-beep; + status = "okay"; + + leds { + compatible = "iei,wt61p803-puzzle-leds"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + led@0 { + reg = <0>; + label = "white:network"; + active-low; + }; + + led@1 { + reg = <1>; + label = "green:cloud"; + active-low; + }; + + led_info: led@2 { + reg = <2>; + label = "orange:info"; + active-low; + }; + + led_power: led@3 { + reg = <3>; + label = "yellow:power"; + active-low; + default-state = "on"; + }; + }; + + hwmon { + compatible = "iei,wt61p803-puzzle-hwmon"; + #address-cells = <1>; + #size-cells = <0>; + + chassis_fan_group0: fan-group@0 { + #cooling-cells = <2>; + reg = <0x00>; + cooling-levels = <64 102 170 230 250>; + }; + }; + }; +}; + +&ap_thermal_cpu1 { + trips { + cpu_active: cpu-active { + temperature = <44000>; + hysteresis = <2000>; + type = "active"; + }; + }; + cooling-maps { + fan-map { + trip = <&cpu_active>; + cooling-device = <&chassis_fan_group0 64 THERMAL_NO_LIMIT>; + }; + }; +}; + +/* on-board eMMC - U9 */ +&ap_sdhci0 { + pinctrl-names = "default"; + bus-width = <8>; + status = "okay"; + mmc-ddr-1_8v; + mmc-hs400-1_8v; +}; + +&cp0_crypto { + status = "okay"; +}; + +&cp0_xmdio { + status = "okay"; + cp0_nbaset_phy0: ethernet-phy@0 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <2>; + }; + cp0_nbaset_phy1: ethernet-phy@1 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <0>; + }; + cp0_nbaset_phy2: ethernet-phy@2 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <8>; + }; +}; + +&cp0_ethernet { + status = "okay"; +}; + +/* SLM-1521-V2, CON9 */ +&cp0_eth0 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp0_comphy2 0>; + phy = <&cp0_nbaset_phy0>; +}; + +&cp0_eth1 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp0_comphy4 1>; + phy = <&cp0_nbaset_phy1>; +}; + +&cp0_eth2 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp0_comphy5 2>; + phy = <&cp0_nbaset_phy2>; +}; + +&cp0_gpio1 { + status = "okay"; +}; + +&cp0_gpio2 { + status = "okay"; +}; + +&cp0_i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&cp0_i2c0_pins>; + status = "okay"; + clock-frequency = <100000>; + rtc@32 { + compatible = "epson,rx8130"; + reg = <0x32>; + wakeup-source; + }; +}; + +/* SLM-1521-V2, CON6 */ +&cp0_pcie0 { + status = "okay"; + num-lanes = <2>; + num-viewport = <8>; + phys = <&cp0_comphy0 0>, <&cp0_comphy1 0>; +}; + +/* U55 */ +&cp0_spi1 { + pinctrl-names = "default"; + pinctrl-0 = <&cp0_spi0_pins>; + reg = <0x700680 0x50>, /* control */ + <0x2000000 0x1000000>; /* CS0 */ + status = "okay"; + spi-flash@0 { + #address-cells = <0x1>; + #size-cells = <0x1>; + compatible = "jedec,spi-nor"; + reg = <0x0>; + spi-max-frequency = <40000000>; + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + partition@0 { + label = "U-Boot"; + reg = <0x0 0x1f0000>; + }; + partition@1f0000 { + label = "U-Boot ENV Factory"; + reg = <0x1f0000 0x10000>; + }; + partition@200000 { + label = "Reserved"; + reg = <0x200000 0x1f0000>; + }; + partition@3f0000 { + label = "U-Boot ENV"; + reg = <0x3f0000 0x10000>; + }; + }; + }; +}; + +&cp0_syscon0 { + cp0_pinctrl: pinctrl { + compatible = "marvell,cp115-standalone-pinctrl"; + cp0_i2c0_pins: cp0-i2c-pins-0 { + marvell,pins = "mpp37", "mpp38"; + marvell,function = "i2c0"; + }; + cp0_i2c1_pins: cp0-i2c-pins-1 { + marvell,pins = "mpp35", "mpp36"; + marvell,function = "i2c1"; + }; + cp0_ge1_rgmii_pins: cp0-ge-rgmii-pins-0 { + marvell,pins = "mpp0", "mpp1", "mpp2", + "mpp3", "mpp4", "mpp5", + "mpp6", "mpp7", "mpp8", + "mpp9", "mpp10", "mpp11"; + marvell,function = "ge0"; + }; + cp0_ge2_rgmii_pins: cp0-ge-rgmii-pins-1 { + marvell,pins = "mpp44", "mpp45", "mpp46", + "mpp47", "mpp48", "mpp49", + "mpp50", "mpp51", "mpp52", + "mpp53", "mpp54", "mpp55"; + marvell,function = "ge1"; + }; + cp0_spi0_pins: cp0-spi-pins-0 { + marvell,pins = "mpp13", "mpp14", "mpp15", "mpp16"; + marvell,function = "spi1"; + }; + }; +}; + +/* + * Instantiate the first connected CP115 + */ + +#define CP11X_NAME cp1 +#define CP11X_BASE f6000000 +#define CP11X_PCIEx_MEM_BASE(iface) (0xe2000000 + (iface * 0x1000000)) +#define CP11X_PCIEx_MEM_SIZE(iface) 0xf00000 +#define CP11X_PCIE0_BASE f6600000 +#define CP11X_PCIE1_BASE f6620000 +#define CP11X_PCIE2_BASE f6640000 + +#include "armada-cp115.dtsi" + +#undef CP11X_NAME +#undef CP11X_BASE +#undef CP11X_PCIEx_MEM_BASE +#undef CP11X_PCIEx_MEM_SIZE +#undef CP11X_PCIE0_BASE +#undef CP11X_PCIE1_BASE +#undef CP11X_PCIE2_BASE + +&cp1_crypto { + status = "okay"; +}; + +&cp1_xmdio { + status = "okay"; + cp1_nbaset_phy0: ethernet-phy@3 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <2>; + }; + cp1_nbaset_phy1: ethernet-phy@4 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <0>; + }; + cp1_nbaset_phy2: ethernet-phy@5 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <8>; + }; +}; + +&cp1_ethernet { + status = "okay"; +}; + +/* CON50 */ +&cp1_eth0 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp1_comphy2 0>; + phy = <&cp1_nbaset_phy0>; +}; + +&cp1_eth1 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp1_comphy4 1>; + phy = <&cp1_nbaset_phy1>; +}; + +&cp1_eth2 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp1_comphy5 2>; + phy = <&cp1_nbaset_phy2>; +}; + +&cp1_sata0 { + status = "okay"; + sata-port@1 { + status = "okay"; + phys = <&cp1_comphy0 1>; + }; +}; + +&cp1_gpio1 { + status = "okay"; +}; + +&cp1_gpio2 { + status = "okay"; +}; + +&cp1_i2c0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cp1_i2c0_pins>; + clock-frequency = <100000>; +}; + +&cp1_syscon0 { + cp1_pinctrl: pinctrl { + compatible = "marvell,cp115-standalone-pinctrl"; + cp1_i2c0_pins: cp1-i2c-pins-0 { + marvell,pins = "mpp37", "mpp38"; + marvell,function = "i2c0"; + }; + cp1_spi0_pins: cp1-spi-pins-0 { + marvell,pins = "mpp13", "mpp14", "mpp15", "mpp16"; + marvell,function = "spi1"; + }; + cp1_xhci0_vbus_pins: cp1-xhci0-vbus-pins { + marvell,pins = "mpp3"; + marvell,function = "gpio"; + }; + cp1_sfp_pins: sfp-pins { + marvell,pins = "mpp8", "mpp9", "mpp10", "mpp11"; + marvell,function = "gpio"; + }; + }; +}; + +&cp1_usb3_1 { + status = "okay"; + phys = <&cp1_comphy3 1>; + phy-names = "usb"; +}; diff --git a/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9131-puzzle-m901.dts b/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9131-puzzle-m901.dts index 80d876b4ad..4c1ca8c104 100644 --- a/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9131-puzzle-m901.dts +++ b/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9131-puzzle-m901.dts @@ -17,6 +17,7 @@ chosen { stdout-path = "serial0:115200n8"; + bootargs-append = " root=/dev/mmcblk0p3"; }; aliases { diff --git a/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9132-db-A.dts b/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9132-db-A.dts new file mode 100644 index 0000000000..cd844552da --- /dev/null +++ b/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9132-db-A.dts @@ -0,0 +1,565 @@ +// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT) +/* + * Copyright (C) 2019 Marvell International Ltd. + * + * Device tree for the CN9132-DB board. + */ + +#include "cn9130.dtsi" + +#include +#include + +/ { + model = "QNAP QHora-322"; + compatible = "qnap,qhora-322", + "marvell,armada-ap807-quad", "marvell,armada-ap807"; + + chosen { + stdout-path = "serial0:115200n8"; + bootargs-append = " root=/dev/mmcblk0p3"; + }; + + aliases { + i2c0 = &cp1_i2c0; + i2c1 = &cp0_i2c0; + gpio1 = &cp0_gpio1; + gpio2 = &cp0_gpio2; + gpio3 = &cp1_gpio1; + gpio4 = &cp1_gpio2; + gpio5 = &cp2_gpio1; + gpio6 = &cp2_gpio2; + ethernet0 = &cp0_eth0; + ethernet1 = &cp0_eth1; + ethernet2 = &cp0_eth2; + ethernet3 = &cp1_eth0; + ethernet4 = &cp1_eth1; + ethernet5 = &cp1_eth2; + ethernet6 = &cp2_eth0; + ethernet7 = &cp2_eth1; + ethernet8 = &cp2_eth2; + spi1 = &cp0_spi0; + spi2 = &cp0_spi1; + led-boot = &led_power; + led-failsafe = &led_info; + led-running = &led_power; + led-upgrade = &led_info; + }; + + memory@00000000 { + device_type = "memory"; + reg = <0x0 0x0 0x0 0x80000000>; + }; + + gpio_keys { + compatible = "gpio-keys"; + + reset { + label = "Reset"; + linux,code = ; + gpios = <&cp0_gpio2 4 GPIO_ACTIVE_LOW>; + }; + }; + + cp2_reg_usb3_vbus0: cp2_usb3_vbus@0 { + compatible = "regulator-fixed"; + regulator-name = "cp2-xhci0-vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&cp2_gpio1 2 GPIO_ACTIVE_HIGH>; + }; + + cp2_usb3_0_phy0: cp2_usb3_phy0 { + compatible = "usb-nop-xceiv"; + vcc-supply = <&cp2_reg_usb3_vbus0>; + }; + + cp2_reg_usb3_vbus1: cp2_usb3_vbus@1 { + compatible = "regulator-fixed"; + regulator-name = "cp2-xhci1-vbus"; + regulator-min-microvolt = <5000000>; + regulator-max-microvolt = <5000000>; + enable-active-high; + gpio = <&cp2_gpio1 3 GPIO_ACTIVE_HIGH>; + }; + + cp2_usb3_0_phy1: cp2_usb3_phy1 { + compatible = "usb-nop-xceiv"; + vcc-supply = <&cp2_reg_usb3_vbus1>; + }; + + cp2_sfp_eth0: sfp-eth0 { + compatible = "sff,sfp"; + i2c-bus = <&cp2_sfpp0_i2c>; + los-gpio = <&cp2_module_expander1 11 GPIO_ACTIVE_HIGH>; + mod-def0-gpio = <&cp2_module_expander1 10 GPIO_ACTIVE_LOW>; + tx-disable-gpio = <&cp2_module_expander1 9 GPIO_ACTIVE_HIGH>; + tx-fault-gpio = <&cp2_module_expander1 8 GPIO_ACTIVE_HIGH>; + status = "disabled"; + }; +}; + +&uart0 { + status = "okay"; +}; + +&cp0_uart0 { + status = "okay"; + + puzzle-mcu { + compatible = "iei,wt61p803-puzzle"; + #address-cells = <1>; + #size-cells = <1>; + current-speed = <115200>; + enable-beep; + status = "okay"; + + leds { + compatible = "iei,wt61p803-puzzle-leds"; + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + led@0 { + reg = <0>; + label = "white:network"; + active-low; + }; + + led@1 { + reg = <1>; + label = "green:cloud"; + active-low; + }; + + led_info: led@2 { + reg = <2>; + label = "orange:info"; + active-low; + }; + + led_power: led@3 { + reg = <3>; + label = "yellow:power"; + active-low; + default-state = "on"; + }; + }; + + hwmon { + compatible = "iei,wt61p803-puzzle-hwmon"; + #address-cells = <1>; + #size-cells = <0>; + + chassis_fan_group0: fan-group@0 { + #cooling-cells = <2>; + reg = <0x00>; + cooling-levels = <64 102 170 230 250>; + }; + }; + }; +}; + +&ap_thermal_cpu1 { + trips { + cpu_active: cpu-active { + temperature = <44000>; + hysteresis = <2000>; + type = "active"; + }; + }; + cooling-maps { + fan-map { + trip = <&cpu_active>; + cooling-device = <&chassis_fan_group0 64 THERMAL_NO_LIMIT>; + }; + }; +}; + +/* on-board eMMC - U9 */ +&ap_sdhci0 { + pinctrl-names = "default"; + bus-width = <8>; + status = "okay"; + mmc-ddr-1_8v; + mmc-hs400-1_8v; +}; + +&cp0_crypto { + status = "okay"; +}; + +&cp0_xmdio { + status = "okay"; + cp0_nbaset_phy0: ethernet-phy@0 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <2>; + }; + cp0_nbaset_phy1: ethernet-phy@1 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <0>; + }; + cp0_nbaset_phy2: ethernet-phy@2 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <8>; + }; +}; + +&cp0_ethernet { + status = "okay"; +}; + +/* SLM-1521-V2, CON9 */ +&cp0_eth0 { + status = "okay"; + phy-mode = "10gbase-kr"; + phys = <&cp0_comphy2 0>; + phy = <&cp0_nbaset_phy0>; +}; + +&cp0_eth1 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp0_comphy4 1>; + phy = <&cp0_nbaset_phy1>; +}; + +&cp0_eth2 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp0_comphy1 2>; + phy = <&cp0_nbaset_phy2>; +}; + +&cp0_gpio1 { + status = "okay"; +}; + +&cp0_gpio2 { + status = "okay"; +}; + +&cp0_i2c0 { + pinctrl-names = "default"; + pinctrl-0 = <&cp0_i2c0_pins>; + status = "okay"; + clock-frequency = <100000>; + rtc@32 { + compatible = "epson,rx8130"; + reg = <0x32>; + wakeup-source; + }; +}; + +&cp0_i2c1 { + clock-frequency = <100000>; +}; + +/* SLM-1521-V2, CON6 */ +&cp0_sata0 { + status = "okay"; + sata-port@1 { + status = "okay"; + phys = <&cp0_comphy0 1>; + }; +}; + +&cp0_pcie2 { + status = "okay"; + num-lanes = <1>; + num-viewport = <8>; + phys = <&cp0_comphy5 2>; +}; + +/* U55 */ +&cp0_spi1 { + pinctrl-names = "default"; + pinctrl-0 = <&cp0_spi0_pins>; + reg = <0x700680 0x50>, /* control */ + <0x2000000 0x1000000>; /* CS0 */ + status = "okay"; + spi-flash@0 { + #address-cells = <0x1>; + #size-cells = <0x1>; + compatible = "jedec,spi-nor"; + reg = <0x0>; + spi-max-frequency = <40000000>; + partitions { + compatible = "fixed-partitions"; + #address-cells = <1>; + #size-cells = <1>; + partition@0 { + label = "U-Boot"; + reg = <0x0 0x1f0000>; + }; + partition@1f0000 { + label = "U-Boot ENV Factory"; + reg = <0x1f0000 0x10000>; + }; + partition@200000 { + label = "Reserved"; + reg = <0x200000 0x1f0000>; + }; + partition@3f0000 { + label = "U-Boot ENV"; + reg = <0x3f0000 0x10000>; + }; + }; + }; +}; + +&cp0_syscon0 { + cp0_pinctrl: pinctrl { + compatible = "marvell,cp115-standalone-pinctrl"; + cp0_i2c0_pins: cp0-i2c-pins-0 { + marvell,pins = "mpp37", "mpp38"; + marvell,function = "i2c0"; + }; + cp0_i2c1_pins: cp0-i2c-pins-1 { + marvell,pins = "mpp35", "mpp36"; + marvell,function = "i2c1"; + }; + cp0_ge1_rgmii_pins: cp0-ge-rgmii-pins-0 { + marvell,pins = "mpp0", "mpp1", "mpp2", + "mpp3", "mpp4", "mpp5", + "mpp6", "mpp7", "mpp8", + "mpp9", "mpp10", "mpp11"; + marvell,function = "ge0"; + }; + cp0_ge2_rgmii_pins: cp0-ge-rgmii-pins-1 { + marvell,pins = "mpp44", "mpp45", "mpp46", + "mpp47", "mpp48", "mpp49", + "mpp50", "mpp51", "mpp52", + "mpp53", "mpp54", "mpp55"; + marvell,function = "ge1"; + }; + cp0_spi0_pins: cp0-spi-pins-0 { + marvell,pins = "mpp13", "mpp14", "mpp15", "mpp16"; + marvell,function = "spi1"; + }; + }; +}; + +&cp0_usb3_1 { + status = "okay"; + phys = <&cp0_comphy3 1>; + phy-names = "usb"; +}; + +/* + * Instantiate the first connected CP115 + */ + +#define CP11X_NAME cp1 +#define CP11X_BASE f4000000 +#define CP11X_PCIEx_MEM_BASE(iface) (0xe2000000 + (iface * 0x1000000)) +#define CP11X_PCIEx_MEM_SIZE(iface) 0xf00000 +#define CP11X_PCIE0_BASE f4600000 +#define CP11X_PCIE1_BASE f4620000 +#define CP11X_PCIE2_BASE f4640000 + +#include "armada-cp115.dtsi" + +#undef CP11X_NAME +#undef CP11X_BASE +#undef CP11X_PCIEx_MEM_BASE +#undef CP11X_PCIEx_MEM_SIZE +#undef CP11X_PCIE0_BASE +#undef CP11X_PCIE1_BASE +#undef CP11X_PCIE2_BASE + +&cp1_crypto { + status = "okay"; +}; + +&cp1_xmdio { + status = "okay"; + cp1_nbaset_phy0: ethernet-phy@3 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <2>; + }; + cp1_nbaset_phy1: ethernet-phy@4 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <0>; + }; + cp1_nbaset_phy2: ethernet-phy@5 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <8>; + }; +}; + +&cp1_ethernet { + status = "okay"; +}; + +/* CON50 */ +&cp1_eth0 { + status = "okay"; + phy-mode = "10gbase-kr"; + phys = <&cp1_comphy2 0>; + phy = <&cp1_nbaset_phy0>; +}; + +&cp1_eth1 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp1_comphy4 1>; + phy = <&cp1_nbaset_phy1>; +}; + +&cp1_eth2 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp1_comphy1 2>; + phy = <&cp1_nbaset_phy2>; +}; + +&cp1_gpio1 { + status = "okay"; +}; + +&cp1_gpio2 { + status = "okay"; +}; + +&cp1_i2c0 { + status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&cp1_i2c0_pins>; + clock-frequency = <100000>; +}; + +&cp1_syscon0 { + cp1_pinctrl: pinctrl { + compatible = "marvell,cp115-standalone-pinctrl"; + cp1_i2c0_pins: cp1-i2c-pins-0 { + marvell,pins = "mpp37", "mpp38"; + marvell,function = "i2c0"; + }; + cp1_spi0_pins: cp1-spi-pins-0 { + marvell,pins = "mpp13", "mpp14", "mpp15", "mpp16"; + marvell,function = "spi1"; + }; + cp1_xhci0_vbus_pins: cp1-xhci0-vbus-pins { + marvell,pins = "mpp3"; + marvell,function = "gpio"; + }; + }; +}; + +/* + * Instantiate the second connected CP115 + */ + +#define CP11X_NAME cp2 +#define CP11X_BASE f6000000 +#define CP11X_PCIEx_MEM_BASE(iface) (0xe5000000 + (iface * 0x1000000)) +#define CP11X_PCIEx_MEM_SIZE(iface) 0xf00000 +#define CP11X_PCIE0_BASE f6600000 +#define CP11X_PCIE1_BASE f6620000 +#define CP11X_PCIE2_BASE f6640000 + +#include "armada-cp115.dtsi" + +#undef CP11X_NAME +#undef CP11X_BASE +#undef CP11X_PCIEx_MEM_BASE +#undef CP11X_PCIEx_MEM_SIZE +#undef CP11X_PCIE0_BASE +#undef CP11X_PCIE1_BASE +#undef CP11X_PCIE2_BASE + +&cp2_crypto { + status = "okay"; +}; + +&cp2_ethernet { + status = "okay"; +}; + +&cp2_xmdio { + status = "okay"; + cp2_nbaset_phy0: ethernet-phy@6 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <2>; + }; + cp2_nbaset_phy1: ethernet-phy@7 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <0>; + }; + cp2_nbaset_phy2: ethernet-phy@8 { + compatible = "ethernet-phy-ieee802.3-c45"; + reg = <8>; + }; +}; + +/* SLM-1521-V2, CON9 */ +&cp2_eth0 { + status = "okay"; + phy-mode = "10gbase-kr"; + phys = <&cp2_comphy2 0>; + phy = <&cp2_nbaset_phy0>; +}; + +&cp2_eth1 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp2_comphy4 1>; + phy = <&cp2_nbaset_phy1>; +}; + +&cp2_eth2 { + status = "okay"; + phy-mode = "2500base-x"; + phys = <&cp2_comphy1 2>; + phy = <&cp2_nbaset_phy2>; +}; + +&cp2_gpio1 { + status = "okay"; +}; + +&cp2_gpio2 { + status = "okay"; +}; + +&cp2_i2c0 { + clock-frequency = <100000>; + /* SLM-1521-V2 - U3 */ + i2c-mux@72 { + compatible = "nxp,pca9544"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0x72>; + cp2_sfpp0_i2c: i2c@0 { + #address-cells = <1>; + #size-cells = <0>; + reg = <0>; + }; + + i2c@1 { + #address-cells = <1>; + #size-cells = <0>; + reg = <1>; + /* U12 */ + cp2_module_expander1: pca9555@21 { + compatible = "nxp,pca9555"; + pinctrl-names = "default"; + gpio-controller; + #gpio-cells = <2>; + reg = <0x21>; + }; + }; + }; +}; + +&cp2_syscon0 { + cp2_pinctrl: pinctrl { + compatible = "marvell,cp115-standalone-pinctrl"; + cp2_i2c0_pins: cp2-i2c-pins-0 { + marvell,pins = "mpp37", "mpp38"; + marvell,function = "i2c0"; + }; + }; +}; diff --git a/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9132-puzzle-m902.dts b/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9132-puzzle-m902.dts index fd99eb2d13..c54fb960ee 100644 --- a/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9132-puzzle-m902.dts +++ b/lede/target/linux/mvebu/files/arch/arm64/boot/dts/marvell/cn9132-puzzle-m902.dts @@ -17,6 +17,7 @@ chosen { stdout-path = "serial0:115200n8"; + bootargs-append = " root=/dev/mmcblk0p3"; }; aliases { diff --git a/lede/target/linux/mvebu/image/Makefile b/lede/target/linux/mvebu/image/Makefile index b0498d34c9..09f8d96351 100644 --- a/lede/target/linux/mvebu/image/Makefile +++ b/lede/target/linux/mvebu/image/Makefile @@ -39,6 +39,16 @@ define Build/boot-img-ext4 $@.bootimg $@.boot endef +define Build/boot-qnap-img-ext4 + rm -fR $@.boot + mkdir -p $@.boot + $(foreach dts,$(DEVICE_DTS), $(CP) $(KDIR)/image-$(dts).dtb $@.boot/$(dts).dtb;) + $(CP) $(KDIR)/vmlinux $@.boot/Image + make_ext4fs -J -L kernel -l $(CONFIG_TARGET_KERNEL_PARTSIZE)M \ + $(if $(SOURCE_DATE_EPOCH),-T $(SOURCE_DATE_EPOCH)) \ + $@.bootimg $@.boot +endef + define Build/buffalo-kernel-jffs2 rm -rf $(KDIR)/kernel_jffs2 $@.fakerd mkdir -p $(KDIR)/kernel_jffs2 diff --git a/lede/target/linux/mvebu/image/cortexa72.mk b/lede/target/linux/mvebu/image/cortexa72.mk index 47d958b2fd..4ef5a47059 100644 --- a/lede/target/linux/mvebu/image/cortexa72.mk +++ b/lede/target/linux/mvebu/image/cortexa72.mk @@ -78,3 +78,15 @@ define Device/iei_puzzle-m902 SOC := cn9132 endef TARGET_DEVICES += iei_puzzle-m902 + +define Device/qnap_qhora-32x + $(call Device/Default-arm64) + SOC := cn9132 + DEVICE_VENDOR := QNAP + DEVICE_MODEL := QHora-321/322 + DEVICE_PACKAGES += kmod-rtc-ds1307 + DEVICE_DTS := cn9131-db-A cn9131-puzzle-m901 cn9132-db-A cn9132-puzzle-m902 + SUPPORTED_DEVICES := qnap,qhora-321 qnap,qhora-322 iei,puzzle-m901 iei,puzzle-m902 + IMAGE/sdcard.img.gz := boot-scr | boot-qnap-img-ext4 | sdcard-img-ext4 | gzip | append-metadata +endef +TARGET_DEVICES += qnap_qhora-32x diff --git a/lede/target/linux/ramips/patches-5.4/993-spi-nor-w25q512jv.patch b/lede/target/linux/ramips/patches-5.4/993-spi-nor-w25q512jv.patch new file mode 100644 index 0000000000..1a16574c2b --- /dev/null +++ b/lede/target/linux/ramips/patches-5.4/993-spi-nor-w25q512jv.patch @@ -0,0 +1,25 @@ +From: David Bauer +Date: Thu, 11 Feb 2021 19:57:26 +0100 +Subject: [PATCH] mtd: spi-nor: add support for Winbond W25Q512 + +The Winbond W25Q512 is a 512mb SPI-NOR chip. It supports 4K sectors as +well as block protection and Dual-/Quad-read. + +Tested on: Ubiquiti UniFi 6 LR + +Signed-off-by: David Bauer + +Ref: https://patchwork.ozlabs.org/project/linux-mtd/patch/20210213151047.11700-1-mail@david-bauer.net/ + +--- a/drivers/mtd/spi-nor/spi-nor.c ++++ b/drivers/mtd/spi-nor/spi-nor.c +@@ -2550,6 +2550,9 @@ static const struct flash_info spi_nor_i + .fixups = &w25q256_fixups }, + { "w25q256jvm", INFO(0xef7019, 0, 64 * 1024, 512, + SECT_4K | SPI_NOR_DUAL_READ | SPI_NOR_QUAD_READ) }, ++ { "w25q512jv", INFO(0xef4020, 0, 64 * 1024, 1024, ++ SECT_4K | SPI_NOR_QUAD_READ | SPI_NOR_DUAL_READ | ++ SPI_NOR_HAS_TB | SPI_NOR_HAS_LOCK) }, + { "w25m512jv", INFO(0xef7119, 0, 64 * 1024, 1024, + SECT_4K | SPI_NOR_QUAD_READ | SPI_NOR_DUAL_READ) }, + diff --git a/ryujinx/src/Ryujinx.Graphics.GAL/Capabilities.cs b/ryujinx/src/Ryujinx.Graphics.GAL/Capabilities.cs index d758586ae4..a5c6eb5c8e 100644 --- a/ryujinx/src/Ryujinx.Graphics.GAL/Capabilities.cs +++ b/ryujinx/src/Ryujinx.Graphics.GAL/Capabilities.cs @@ -51,6 +51,13 @@ namespace Ryujinx.Graphics.GAL public readonly bool SupportsIndirectParameters; public readonly bool SupportsDepthClipControl; + public readonly int UniformBufferSetIndex; + public readonly int StorageBufferSetIndex; + public readonly int TextureSetIndex; + public readonly int ImageSetIndex; + public readonly int ExtraSetBaseIndex; + public readonly int MaximumExtraSets; + public readonly uint MaximumUniformBuffersPerStage; public readonly uint MaximumStorageBuffersPerStage; public readonly uint MaximumTexturesPerStage; @@ -109,6 +116,12 @@ namespace Ryujinx.Graphics.GAL bool supportsViewportSwizzle, bool supportsIndirectParameters, bool supportsDepthClipControl, + int uniformBufferSetIndex, + int storageBufferSetIndex, + int textureSetIndex, + int imageSetIndex, + int extraSetBaseIndex, + int maximumExtraSets, uint maximumUniformBuffersPerStage, uint maximumStorageBuffersPerStage, uint maximumTexturesPerStage, @@ -164,6 +177,12 @@ namespace Ryujinx.Graphics.GAL SupportsViewportSwizzle = supportsViewportSwizzle; SupportsIndirectParameters = supportsIndirectParameters; SupportsDepthClipControl = supportsDepthClipControl; + UniformBufferSetIndex = uniformBufferSetIndex; + StorageBufferSetIndex = storageBufferSetIndex; + TextureSetIndex = textureSetIndex; + ImageSetIndex = imageSetIndex; + ExtraSetBaseIndex = extraSetBaseIndex; + MaximumExtraSets = maximumExtraSets; MaximumUniformBuffersPerStage = maximumUniformBuffersPerStage; MaximumStorageBuffersPerStage = maximumStorageBuffersPerStage; MaximumTexturesPerStage = maximumTexturesPerStage; diff --git a/ryujinx/src/Ryujinx.Graphics.GAL/IPipeline.cs b/ryujinx/src/Ryujinx.Graphics.GAL/IPipeline.cs index 9efb9e3e8f..cbf1bc3a22 100644 --- a/ryujinx/src/Ryujinx.Graphics.GAL/IPipeline.cs +++ b/ryujinx/src/Ryujinx.Graphics.GAL/IPipeline.cs @@ -60,6 +60,7 @@ namespace Ryujinx.Graphics.GAL void SetImage(ShaderStage stage, int binding, ITexture texture, Format imageFormat); void SetImageArray(ShaderStage stage, int binding, IImageArray array); + void SetImageArraySeparate(ShaderStage stage, int setIndex, IImageArray array); void SetLineParameters(float width, bool smooth); @@ -91,6 +92,7 @@ namespace Ryujinx.Graphics.GAL void SetTextureAndSampler(ShaderStage stage, int binding, ITexture texture, ISampler sampler); void SetTextureArray(ShaderStage stage, int binding, ITextureArray array); + void SetTextureArraySeparate(ShaderStage stage, int setIndex, ITextureArray array); void SetTransformFeedbackBuffers(ReadOnlySpan buffers); void SetUniformBuffers(ReadOnlySpan buffers); diff --git a/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs index 23f1a64ef9..edaae3042d 100644 --- a/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs +++ b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs @@ -124,6 +124,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading Register(CommandType.SetUniformBuffers); Register(CommandType.SetImage); Register(CommandType.SetImageArray); + Register(CommandType.SetImageArraySeparate); Register(CommandType.SetIndexBuffer); Register(CommandType.SetLineParameters); Register(CommandType.SetLogicOpState); @@ -141,6 +142,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading Register(CommandType.SetStencilTest); Register(CommandType.SetTextureAndSampler); Register(CommandType.SetTextureArray); + Register(CommandType.SetTextureArraySeparate); Register(CommandType.SetUserClipDistance); Register(CommandType.SetVertexAttribs); Register(CommandType.SetVertexBuffers); diff --git a/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs index f95aab05b4..7586953526 100644 --- a/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs +++ b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs @@ -84,6 +84,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading SetUniformBuffers, SetImage, SetImageArray, + SetImageArraySeparate, SetIndexBuffer, SetLineParameters, SetLogicOpState, @@ -101,6 +102,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading SetStencilTest, SetTextureAndSampler, SetTextureArray, + SetTextureArraySeparate, SetUserClipDistance, SetVertexAttribs, SetVertexBuffers, diff --git a/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/Commands/SetImageArraySeparateCommand.cs b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/Commands/SetImageArraySeparateCommand.cs new file mode 100644 index 0000000000..abeb58a06f --- /dev/null +++ b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/Commands/SetImageArraySeparateCommand.cs @@ -0,0 +1,26 @@ +using Ryujinx.Graphics.GAL.Multithreading.Model; +using Ryujinx.Graphics.GAL.Multithreading.Resources; +using Ryujinx.Graphics.Shader; + +namespace Ryujinx.Graphics.GAL.Multithreading.Commands +{ + struct SetImageArraySeparateCommand : IGALCommand, IGALCommand + { + public readonly CommandType CommandType => CommandType.SetImageArraySeparate; + private ShaderStage _stage; + private int _setIndex; + private TableRef _array; + + public void Set(ShaderStage stage, int setIndex, TableRef array) + { + _stage = stage; + _setIndex = setIndex; + _array = array; + } + + public static void Run(ref SetImageArraySeparateCommand command, ThreadedRenderer threaded, IRenderer renderer) + { + renderer.Pipeline.SetImageArraySeparate(command._stage, command._setIndex, command._array.GetAs(threaded)?.Base); + } + } +} diff --git a/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/Commands/SetTextureArraySeparateCommand.cs b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/Commands/SetTextureArraySeparateCommand.cs new file mode 100644 index 0000000000..b179f2e70a --- /dev/null +++ b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/Commands/SetTextureArraySeparateCommand.cs @@ -0,0 +1,26 @@ +using Ryujinx.Graphics.GAL.Multithreading.Model; +using Ryujinx.Graphics.GAL.Multithreading.Resources; +using Ryujinx.Graphics.Shader; + +namespace Ryujinx.Graphics.GAL.Multithreading.Commands +{ + struct SetTextureArraySeparateCommand : IGALCommand, IGALCommand + { + public readonly CommandType CommandType => CommandType.SetTextureArraySeparate; + private ShaderStage _stage; + private int _setIndex; + private TableRef _array; + + public void Set(ShaderStage stage, int setIndex, TableRef array) + { + _stage = stage; + _setIndex = setIndex; + _array = array; + } + + public static void Run(ref SetTextureArraySeparateCommand command, ThreadedRenderer threaded, IRenderer renderer) + { + renderer.Pipeline.SetTextureArraySeparate(command._stage, command._setIndex, command._array.GetAs(threaded)?.Base); + } + } +} diff --git a/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs index 697894eb52..edd79d8a07 100644 --- a/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs +++ b/ryujinx/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedPipeline.cs @@ -189,6 +189,12 @@ namespace Ryujinx.Graphics.GAL.Multithreading _renderer.QueueCommand(); } + public void SetImageArraySeparate(ShaderStage stage, int setIndex, IImageArray array) + { + _renderer.New().Set(stage, setIndex, Ref(array)); + _renderer.QueueCommand(); + } + public void SetIndexBuffer(BufferRange buffer, IndexType type) { _renderer.New().Set(buffer, type); @@ -297,6 +303,12 @@ namespace Ryujinx.Graphics.GAL.Multithreading _renderer.QueueCommand(); } + public void SetTextureArraySeparate(ShaderStage stage, int setIndex, ITextureArray array) + { + _renderer.New().Set(stage, setIndex, Ref(array)); + _renderer.QueueCommand(); + } + public void SetTransformFeedbackBuffers(ReadOnlySpan buffers) { _renderer.New().Set(_renderer.CopySpan(buffers)); diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Image/TextureBindingInfo.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Image/TextureBindingInfo.cs index ba895c60a8..31abc21e8a 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Image/TextureBindingInfo.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Image/TextureBindingInfo.cs @@ -19,6 +19,11 @@ namespace Ryujinx.Graphics.Gpu.Image /// public Format Format { get; } + /// + /// Shader texture host set index. + /// + public int Set { get; } + /// /// Shader texture host binding point. /// @@ -54,15 +59,17 @@ namespace Ryujinx.Graphics.Gpu.Image /// /// The shader sampler target type /// Format of the image as declared on the shader + /// Shader texture host set index /// The shader texture binding point /// For array of textures, this indicates the length of the array. A value of one indicates it is not an array /// Constant buffer slot where the texture handle is located /// The shader texture handle (read index into the texture constant buffer) /// The texture's usage flags, indicating how it is used in the shader - public TextureBindingInfo(Target target, Format format, int binding, int arrayLength, int cbufSlot, int handle, TextureUsageFlags flags) + public TextureBindingInfo(Target target, Format format, int set, int binding, int arrayLength, int cbufSlot, int handle, TextureUsageFlags flags) { Target = target; Format = format; + Set = set; Binding = binding; ArrayLength = arrayLength; CbufSlot = cbufSlot; @@ -74,6 +81,7 @@ namespace Ryujinx.Graphics.Gpu.Image /// Constructs the texture binding information structure. /// /// The shader sampler target type + /// Shader texture host set index /// The shader texture binding point /// For array of textures, this indicates the length of the array. A value of one indicates it is not an array /// Constant buffer slot where the texture handle is located @@ -82,12 +90,13 @@ namespace Ryujinx.Graphics.Gpu.Image /// Indicates that the binding is for a sampler public TextureBindingInfo( Target target, + int set, int binding, int arrayLength, int cbufSlot, int handle, TextureUsageFlags flags, - bool isSamplerOnly) : this(target, 0, binding, arrayLength, cbufSlot, handle, flags) + bool isSamplerOnly) : this(target, 0, set, binding, arrayLength, cbufSlot, handle, flags) { IsSamplerOnly = isSamplerOnly; } diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs index a54d070004..18e28b3dd7 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsArrayCache.cs @@ -566,7 +566,7 @@ namespace Ryujinx.Graphics.Gpu.Image int stageIndex, int textureBufferIndex, SamplerIndex samplerIndex, - TextureBindingInfo bindingInfo) + in TextureBindingInfo bindingInfo) { Update(texturePool, samplerPool, stage, stageIndex, textureBufferIndex, isImage: false, samplerIndex, bindingInfo); } @@ -579,7 +579,7 @@ namespace Ryujinx.Graphics.Gpu.Image /// Shader stage index where the array is used /// Texture constant buffer index /// Array binding information - public void UpdateImageArray(TexturePool texturePool, ShaderStage stage, int stageIndex, int textureBufferIndex, TextureBindingInfo bindingInfo) + public void UpdateImageArray(TexturePool texturePool, ShaderStage stage, int stageIndex, int textureBufferIndex, in TextureBindingInfo bindingInfo) { Update(texturePool, null, stage, stageIndex, textureBufferIndex, isImage: true, SamplerIndex.ViaHeaderIndex, bindingInfo); } @@ -603,7 +603,7 @@ namespace Ryujinx.Graphics.Gpu.Image int textureBufferIndex, bool isImage, SamplerIndex samplerIndex, - TextureBindingInfo bindingInfo) + in TextureBindingInfo bindingInfo) { if (IsDirectHandleType(bindingInfo.Handle)) { @@ -623,7 +623,7 @@ namespace Ryujinx.Graphics.Gpu.Image /// Shader stage where the array is used /// Whether the array is a image or texture array /// Array binding information - private void UpdateFromPool(TexturePool texturePool, SamplerPool samplerPool, ShaderStage stage, bool isImage, TextureBindingInfo bindingInfo) + private void UpdateFromPool(TexturePool texturePool, SamplerPool samplerPool, ShaderStage stage, bool isImage, in TextureBindingInfo bindingInfo) { CacheEntry entry = GetOrAddEntry(texturePool, samplerPool, bindingInfo, isImage, out bool isNewEntry); @@ -638,11 +638,11 @@ namespace Ryujinx.Graphics.Gpu.Image if (isImage) { - _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); + SetImageArray(stage, bindingInfo, entry.ImageArray); } else { - _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); + SetTextureArray(stage, bindingInfo, entry.TextureArray); } return; @@ -737,14 +737,14 @@ namespace Ryujinx.Graphics.Gpu.Image entry.ImageArray.SetFormats(0, formats); entry.ImageArray.SetImages(0, textures); - _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); + SetImageArray(stage, bindingInfo, entry.ImageArray); } else { entry.TextureArray.SetSamplers(0, samplers); entry.TextureArray.SetTextures(0, textures); - _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); + SetTextureArray(stage, bindingInfo, entry.TextureArray); } } @@ -767,7 +767,7 @@ namespace Ryujinx.Graphics.Gpu.Image int textureBufferIndex, bool isImage, SamplerIndex samplerIndex, - TextureBindingInfo bindingInfo) + in TextureBindingInfo bindingInfo) { (textureBufferIndex, int samplerBufferIndex) = TextureHandle.UnpackSlots(bindingInfo.CbufSlot, textureBufferIndex); @@ -800,11 +800,11 @@ namespace Ryujinx.Graphics.Gpu.Image if (isImage) { - _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); + SetImageArray(stage, bindingInfo, entry.ImageArray); } else { - _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); + SetTextureArray(stage, bindingInfo, entry.TextureArray); } return; @@ -829,11 +829,11 @@ namespace Ryujinx.Graphics.Gpu.Image if (isImage) { - _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); + SetImageArray(stage, bindingInfo, entry.ImageArray); } else { - _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); + SetTextureArray(stage, bindingInfo, entry.TextureArray); } return; @@ -950,14 +950,50 @@ namespace Ryujinx.Graphics.Gpu.Image entry.ImageArray.SetFormats(0, formats); entry.ImageArray.SetImages(0, textures); - _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, entry.ImageArray); + SetImageArray(stage, bindingInfo, entry.ImageArray); } else { entry.TextureArray.SetSamplers(0, samplers); entry.TextureArray.SetTextures(0, textures); - _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, entry.TextureArray); + SetTextureArray(stage, bindingInfo, entry.TextureArray); + } + } + + /// + /// Updates a texture array binding on the host. + /// + /// Shader stage where the array is used + /// Array binding information + /// Texture array + private void SetTextureArray(ShaderStage stage, in TextureBindingInfo bindingInfo, ITextureArray array) + { + if (bindingInfo.Set >= _context.Capabilities.ExtraSetBaseIndex && _context.Capabilities.MaximumExtraSets != 0) + { + _context.Renderer.Pipeline.SetTextureArraySeparate(stage, bindingInfo.Set, array); + } + else + { + _context.Renderer.Pipeline.SetTextureArray(stage, bindingInfo.Binding, array); + } + } + + /// + /// Updates a image array binding on the host. + /// + /// Shader stage where the array is used + /// Array binding information + /// Image array + private void SetImageArray(ShaderStage stage, in TextureBindingInfo bindingInfo, IImageArray array) + { + if (bindingInfo.Set >= _context.Capabilities.ExtraSetBaseIndex && _context.Capabilities.MaximumExtraSets != 0) + { + _context.Renderer.Pipeline.SetImageArraySeparate(stage, bindingInfo.Set, array); + } + else + { + _context.Renderer.Pipeline.SetImageArray(stage, bindingInfo.Binding, array); } } @@ -973,7 +1009,7 @@ namespace Ryujinx.Graphics.Gpu.Image private CacheEntry GetOrAddEntry( TexturePool texturePool, SamplerPool samplerPool, - TextureBindingInfo bindingInfo, + in TextureBindingInfo bindingInfo, bool isImage, out bool isNew) { @@ -1015,7 +1051,7 @@ namespace Ryujinx.Graphics.Gpu.Image private CacheEntryFromBuffer GetOrAddEntry( TexturePool texturePool, SamplerPool samplerPool, - TextureBindingInfo bindingInfo, + in TextureBindingInfo bindingInfo, bool isImage, ref BufferBounds textureBufferBounds, out bool isNew) diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/CachedShaderBindings.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/CachedShaderBindings.cs index a80dcbc875..51be00b6e7 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/CachedShaderBindings.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/CachedShaderBindings.cs @@ -62,6 +62,7 @@ namespace Ryujinx.Graphics.Gpu.Shader var result = new TextureBindingInfo( target, + descriptor.Set, descriptor.Binding, descriptor.ArrayLength, descriptor.CbufSlot, @@ -90,6 +91,7 @@ namespace Ryujinx.Graphics.Gpu.Shader var result = new TextureBindingInfo( target, format, + descriptor.Set, descriptor.Binding, descriptor.ArrayLength, descriptor.CbufSlot, diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/BinarySerializer.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/BinarySerializer.cs index c4a648fe4d..ab4508f6d1 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/BinarySerializer.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/BinarySerializer.cs @@ -125,9 +125,18 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache CompressionAlgorithm algorithm = CompressionAlgorithm.None; Read(ref algorithm); - if (algorithm == CompressionAlgorithm.Deflate) + switch (algorithm) { - _activeStream = new DeflateStream(_stream, CompressionMode.Decompress, true); + case CompressionAlgorithm.None: + break; + case CompressionAlgorithm.Deflate: + _activeStream = new DeflateStream(_stream, CompressionMode.Decompress, true); + break; + case CompressionAlgorithm.Brotli: + _activeStream = new BrotliStream(_stream, CompressionMode.Decompress, true); + break; + default: + throw new ArgumentException($"Invalid compression algorithm \"{algorithm}\""); } } @@ -139,9 +148,18 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache { Write(ref algorithm); - if (algorithm == CompressionAlgorithm.Deflate) + switch (algorithm) { - _activeStream = new DeflateStream(_stream, CompressionLevel.Fastest, true); + case CompressionAlgorithm.None: + break; + case CompressionAlgorithm.Deflate: + _activeStream = new DeflateStream(_stream, CompressionLevel.Fastest, true); + break; + case CompressionAlgorithm.Brotli: + _activeStream = new BrotliStream(_stream, CompressionLevel.Fastest, true); + break; + default: + throw new ArgumentException($"Invalid compression algorithm \"{algorithm}\""); } } @@ -187,6 +205,14 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache } stream.Dispose(); break; + case CompressionAlgorithm.Brotli: + stream = new BrotliStream(stream, CompressionMode.Decompress, true); + for (int offset = 0; offset < data.Length;) + { + offset += stream.Read(data[offset..]); + } + stream.Dispose(); + break; } } @@ -210,6 +236,11 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache stream.Write(data); stream.Dispose(); break; + case CompressionAlgorithm.Brotli: + stream = new BrotliStream(stream, CompressionLevel.Fastest, true); + stream.Write(data); + stream.Dispose(); + break; } } } diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/CompressionAlgorithm.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/CompressionAlgorithm.cs index 96ddbb513e..86d3de07de 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/CompressionAlgorithm.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/CompressionAlgorithm.cs @@ -14,5 +14,10 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache /// Deflate compression (RFC 1951). /// Deflate, + + /// + /// Brotli compression (RFC 7932). + /// + Brotli, } } diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheCommon.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheCommon.cs index c4ce0b8702..cecfe9acf8 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheCommon.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheCommon.cs @@ -51,7 +51,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache /// Compression algorithm public static CompressionAlgorithm GetCompressionAlgorithm() { - return CompressionAlgorithm.Deflate; + return CompressionAlgorithm.Brotli; } } } diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs index ea54049c2a..fbf48f017e 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/DiskCache/DiskCacheHostStorage.cs @@ -22,7 +22,7 @@ namespace Ryujinx.Graphics.Gpu.Shader.DiskCache private const ushort FileFormatVersionMajor = 1; private const ushort FileFormatVersionMinor = 2; private const uint FileFormatVersionPacked = ((uint)FileFormatVersionMajor << 16) | FileFormatVersionMinor; - private const uint CodeGenVersion = 5936; + private const uint CodeGenVersion = 6852; private const string SharedTocFileName = "shared.toc"; private const string SharedDataFileName = "shared.data"; diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs index 0d562b0da2..d89eebabfd 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/GpuAccessorBase.cs @@ -51,7 +51,7 @@ namespace Ryujinx.Graphics.Gpu.Shader _reservedImages = rrc.ReservedImages; } - public int CreateConstantBufferBinding(int index) + public SetBindingPair CreateConstantBufferBinding(int index) { int binding; @@ -64,10 +64,10 @@ namespace Ryujinx.Graphics.Gpu.Shader binding = _resourceCounts.UniformBuffersCount++; } - return binding + _reservedConstantBuffers; + return new SetBindingPair(_context.Capabilities.UniformBufferSetIndex, binding + _reservedConstantBuffers); } - public int CreateImageBinding(int count, bool isBuffer) + public SetBindingPair CreateImageBinding(int count, bool isBuffer) { int binding; @@ -96,10 +96,10 @@ namespace Ryujinx.Graphics.Gpu.Shader _resourceCounts.ImagesCount += count; } - return binding + _reservedImages; + return new SetBindingPair(_context.Capabilities.ImageSetIndex, binding + _reservedImages); } - public int CreateStorageBufferBinding(int index) + public SetBindingPair CreateStorageBufferBinding(int index) { int binding; @@ -112,10 +112,10 @@ namespace Ryujinx.Graphics.Gpu.Shader binding = _resourceCounts.StorageBuffersCount++; } - return binding + _reservedStorageBuffers; + return new SetBindingPair(_context.Capabilities.StorageBufferSetIndex, binding + _reservedStorageBuffers); } - public int CreateTextureBinding(int count, bool isBuffer) + public SetBindingPair CreateTextureBinding(int count, bool isBuffer) { int binding; @@ -144,7 +144,7 @@ namespace Ryujinx.Graphics.Gpu.Shader _resourceCounts.TexturesCount += count; } - return binding + _reservedTextures; + return new SetBindingPair(_context.Capabilities.TextureSetIndex, binding + _reservedTextures); } private int GetBindingFromIndex(int index, uint maxPerStage, string resourceName) @@ -183,6 +183,16 @@ namespace Ryujinx.Graphics.Gpu.Shader return maxPerStage * Constants.ShaderStages; } + public int CreateExtraSet() + { + if (_resourceCounts.SetsCount >= _context.Capabilities.MaximumExtraSets) + { + return -1; + } + + return _context.Capabilities.ExtraSetBaseIndex + _resourceCounts.SetsCount++; + } + public int QueryHostGatherBiasPrecision() => _context.Capabilities.GatherBiasPrecision; public bool QueryHostReducedPrecision() => _context.Capabilities.ReduceShaderPrecision; diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/ResourceCounts.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/ResourceCounts.cs index 126e3249c0..59ab378cf8 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/ResourceCounts.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/ResourceCounts.cs @@ -24,5 +24,10 @@ namespace Ryujinx.Graphics.Gpu.Shader /// Total of images used by the shaders. /// public int ImagesCount; + + /// + /// Total of extra sets used by the shaders. + /// + public int SetsCount; } } diff --git a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/ShaderInfoBuilder.cs b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/ShaderInfoBuilder.cs index ed56db3b30..42b2cbb59b 100644 --- a/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/ShaderInfoBuilder.cs +++ b/ryujinx/src/Ryujinx.Graphics.Gpu/Shader/ShaderInfoBuilder.cs @@ -1,5 +1,6 @@ using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Shader; +using System; using System.Collections.Generic; namespace Ryujinx.Graphics.Gpu.Shader @@ -9,13 +10,6 @@ namespace Ryujinx.Graphics.Gpu.Shader /// class ShaderInfoBuilder { - private const int TotalSets = 4; - - private const int UniformSetIndex = 0; - private const int StorageSetIndex = 1; - private const int TextureSetIndex = 2; - private const int ImageSetIndex = 3; - private const ResourceStages SupportBufferStages = ResourceStages.Compute | ResourceStages.Vertex | @@ -36,8 +30,8 @@ namespace Ryujinx.Graphics.Gpu.Shader private readonly int _reservedTextures; private readonly int _reservedImages; - private readonly List[] _resourceDescriptors; - private readonly List[] _resourceUsages; + private List[] _resourceDescriptors; + private List[] _resourceUsages; /// /// Creates a new shader info builder. @@ -51,17 +45,27 @@ namespace Ryujinx.Graphics.Gpu.Shader _fragmentOutputMap = -1; - _resourceDescriptors = new List[TotalSets]; - _resourceUsages = new List[TotalSets]; + int uniformSetIndex = context.Capabilities.UniformBufferSetIndex; + int storageSetIndex = context.Capabilities.StorageBufferSetIndex; + int textureSetIndex = context.Capabilities.TextureSetIndex; + int imageSetIndex = context.Capabilities.ImageSetIndex; - for (int index = 0; index < TotalSets; index++) + int totalSets = Math.Max(uniformSetIndex, storageSetIndex); + totalSets = Math.Max(totalSets, textureSetIndex); + totalSets = Math.Max(totalSets, imageSetIndex); + totalSets++; + + _resourceDescriptors = new List[totalSets]; + _resourceUsages = new List[totalSets]; + + for (int index = 0; index < totalSets; index++) { _resourceDescriptors[index] = new(); _resourceUsages[index] = new(); } - AddDescriptor(SupportBufferStages, ResourceType.UniformBuffer, UniformSetIndex, 0, 1); - AddUsage(SupportBufferStages, ResourceType.UniformBuffer, UniformSetIndex, 0, 1); + AddDescriptor(SupportBufferStages, ResourceType.UniformBuffer, uniformSetIndex, 0, 1); + AddUsage(SupportBufferStages, ResourceType.UniformBuffer, uniformSetIndex, 0, 1); ResourceReservationCounts rrc = new(!context.Capabilities.SupportsTransformFeedback && tfEnabled, vertexAsCompute); @@ -73,12 +77,20 @@ namespace Ryujinx.Graphics.Gpu.Shader // TODO: Handle that better? Maybe we should only set the binding that are really needed on each shader. ResourceStages stages = vertexAsCompute ? ResourceStages.Compute | ResourceStages.Vertex : VtgStages; - PopulateDescriptorAndUsages(stages, ResourceType.UniformBuffer, UniformSetIndex, 1, rrc.ReservedConstantBuffers - 1); - PopulateDescriptorAndUsages(stages, ResourceType.StorageBuffer, StorageSetIndex, 0, rrc.ReservedStorageBuffers); - PopulateDescriptorAndUsages(stages, ResourceType.BufferTexture, TextureSetIndex, 0, rrc.ReservedTextures); - PopulateDescriptorAndUsages(stages, ResourceType.BufferImage, ImageSetIndex, 0, rrc.ReservedImages); + PopulateDescriptorAndUsages(stages, ResourceType.UniformBuffer, uniformSetIndex, 1, rrc.ReservedConstantBuffers - 1); + PopulateDescriptorAndUsages(stages, ResourceType.StorageBuffer, storageSetIndex, 0, rrc.ReservedStorageBuffers); + PopulateDescriptorAndUsages(stages, ResourceType.BufferTexture, textureSetIndex, 0, rrc.ReservedTextures); + PopulateDescriptorAndUsages(stages, ResourceType.BufferImage, imageSetIndex, 0, rrc.ReservedImages); } + /// + /// Populates descriptors and usages for vertex as compute and transform feedback emulation reserved resources. + /// + /// Shader stages where the resources are used + /// Resource type + /// Resource set index where the resources are used + /// First binding number + /// Amount of bindings private void PopulateDescriptorAndUsages(ResourceStages stages, ResourceType type, int setIndex, int start, int count) { AddDescriptor(stages, type, setIndex, start, count); @@ -127,18 +139,23 @@ namespace Ryujinx.Graphics.Gpu.Shader int textureBinding = _reservedTextures + stageIndex * texturesPerStage * 2; int imageBinding = _reservedImages + stageIndex * imagesPerStage * 2; - AddDescriptor(stages, ResourceType.UniformBuffer, UniformSetIndex, uniformBinding, uniformsPerStage); - AddDescriptor(stages, ResourceType.StorageBuffer, StorageSetIndex, storageBinding, storagesPerStage); - AddDualDescriptor(stages, ResourceType.TextureAndSampler, ResourceType.BufferTexture, TextureSetIndex, textureBinding, texturesPerStage); - AddDualDescriptor(stages, ResourceType.Image, ResourceType.BufferImage, ImageSetIndex, imageBinding, imagesPerStage); + int uniformSetIndex = _context.Capabilities.UniformBufferSetIndex; + int storageSetIndex = _context.Capabilities.StorageBufferSetIndex; + int textureSetIndex = _context.Capabilities.TextureSetIndex; + int imageSetIndex = _context.Capabilities.ImageSetIndex; - AddArrayDescriptors(info.Textures, stages, TextureSetIndex, isImage: false); - AddArrayDescriptors(info.Images, stages, TextureSetIndex, isImage: true); + AddDescriptor(stages, ResourceType.UniformBuffer, uniformSetIndex, uniformBinding, uniformsPerStage); + AddDescriptor(stages, ResourceType.StorageBuffer, storageSetIndex, storageBinding, storagesPerStage); + AddDualDescriptor(stages, ResourceType.TextureAndSampler, ResourceType.BufferTexture, textureSetIndex, textureBinding, texturesPerStage); + AddDualDescriptor(stages, ResourceType.Image, ResourceType.BufferImage, imageSetIndex, imageBinding, imagesPerStage); - AddUsage(info.CBuffers, stages, UniformSetIndex, isStorage: false); - AddUsage(info.SBuffers, stages, StorageSetIndex, isStorage: true); - AddUsage(info.Textures, stages, TextureSetIndex, isImage: false); - AddUsage(info.Images, stages, ImageSetIndex, isImage: true); + AddArrayDescriptors(info.Textures, stages, isImage: false); + AddArrayDescriptors(info.Images, stages, isImage: true); + + AddUsage(info.CBuffers, stages, isStorage: false); + AddUsage(info.SBuffers, stages, isStorage: true); + AddUsage(info.Textures, stages, isImage: false); + AddUsage(info.Images, stages, isImage: true); } /// @@ -177,9 +194,8 @@ namespace Ryujinx.Graphics.Gpu.Shader /// /// Textures to be added /// Stages where the textures are used - /// Descriptor set index where the textures will be bound /// True for images, false for textures - private void AddArrayDescriptors(IEnumerable textures, ResourceStages stages, int setIndex, bool isImage) + private void AddArrayDescriptors(IEnumerable textures, ResourceStages stages, bool isImage) { foreach (TextureDescriptor texture in textures) { @@ -187,7 +203,7 @@ namespace Ryujinx.Graphics.Gpu.Shader { ResourceType type = GetTextureResourceType(texture, isImage); - _resourceDescriptors[setIndex].Add(new ResourceDescriptor(texture.Binding, texture.ArrayLength, type, stages)); + GetDescriptors(texture.Set).Add(new ResourceDescriptor(texture.Binding, texture.ArrayLength, type, stages)); } } } @@ -213,13 +229,12 @@ namespace Ryujinx.Graphics.Gpu.Shader /// /// Buffers to be added /// Stages where the buffers are used - /// Descriptor set index where the buffers will be bound /// True for storage buffers, false for uniform buffers - private void AddUsage(IEnumerable buffers, ResourceStages stages, int setIndex, bool isStorage) + private void AddUsage(IEnumerable buffers, ResourceStages stages, bool isStorage) { foreach (BufferDescriptor buffer in buffers) { - _resourceUsages[setIndex].Add(new ResourceUsage( + GetUsages(buffer.Set).Add(new ResourceUsage( buffer.Binding, 1, isStorage ? ResourceType.StorageBuffer : ResourceType.UniformBuffer, @@ -232,18 +247,65 @@ namespace Ryujinx.Graphics.Gpu.Shader /// /// Textures to be added /// Stages where the textures are used - /// Descriptor set index where the textures will be bound /// True for images, false for textures - private void AddUsage(IEnumerable textures, ResourceStages stages, int setIndex, bool isImage) + private void AddUsage(IEnumerable textures, ResourceStages stages, bool isImage) { foreach (TextureDescriptor texture in textures) { ResourceType type = GetTextureResourceType(texture, isImage); - _resourceUsages[setIndex].Add(new ResourceUsage(texture.Binding, texture.ArrayLength, type, stages)); + GetUsages(texture.Set).Add(new ResourceUsage(texture.Binding, texture.ArrayLength, type, stages)); } } + /// + /// Gets the list of resource descriptors for a given set index. A new list will be created if needed. + /// + /// Resource set index + /// List of resource descriptors + private List GetDescriptors(int setIndex) + { + if (_resourceDescriptors.Length <= setIndex) + { + int oldLength = _resourceDescriptors.Length; + Array.Resize(ref _resourceDescriptors, setIndex + 1); + + for (int index = oldLength; index <= setIndex; index++) + { + _resourceDescriptors[index] = new(); + } + } + + return _resourceDescriptors[setIndex]; + } + + /// + /// Gets the list of resource usages for a given set index. A new list will be created if needed. + /// + /// Resource set index + /// List of resource usages + private List GetUsages(int setIndex) + { + if (_resourceUsages.Length <= setIndex) + { + int oldLength = _resourceUsages.Length; + Array.Resize(ref _resourceUsages, setIndex + 1); + + for (int index = oldLength; index <= setIndex; index++) + { + _resourceUsages[index] = new(); + } + } + + return _resourceUsages[setIndex]; + } + + /// + /// Gets a resource type from a texture descriptor. + /// + /// Texture descriptor + /// Whether the texture is a image texture (writable) or not (sampled) + /// Resource type private static ResourceType GetTextureResourceType(TextureDescriptor texture, bool isImage) { bool isBuffer = (texture.Type & SamplerType.Mask) == SamplerType.TextureBuffer; @@ -278,10 +340,12 @@ namespace Ryujinx.Graphics.Gpu.Shader /// Shader information public ShaderInfo Build(ProgramPipelineState? pipeline, bool fromCache = false) { - var descriptors = new ResourceDescriptorCollection[TotalSets]; - var usages = new ResourceUsageCollection[TotalSets]; + int totalSets = _resourceDescriptors.Length; - for (int index = 0; index < TotalSets; index++) + var descriptors = new ResourceDescriptorCollection[totalSets]; + var usages = new ResourceUsageCollection[totalSets]; + + for (int index = 0; index < totalSets; index++) { descriptors[index] = new ResourceDescriptorCollection(_resourceDescriptors[index].ToArray().AsReadOnly()); usages[index] = new ResourceUsageCollection(_resourceUsages[index].ToArray().AsReadOnly()); diff --git a/ryujinx/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs b/ryujinx/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs index 7bcff947e4..ba9cd45c67 100644 --- a/ryujinx/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs +++ b/ryujinx/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs @@ -187,6 +187,12 @@ namespace Ryujinx.Graphics.OpenGL supportsViewportSwizzle: HwCapabilities.SupportsViewportSwizzle, supportsIndirectParameters: HwCapabilities.SupportsIndirectParameters, supportsDepthClipControl: true, + uniformBufferSetIndex: 0, + storageBufferSetIndex: 1, + textureSetIndex: 2, + imageSetIndex: 3, + extraSetBaseIndex: 0, + maximumExtraSets: 0, maximumUniformBuffersPerStage: 13, // TODO: Avoid hardcoding those limits here and get from driver? maximumStorageBuffersPerStage: 16, maximumTexturesPerStage: 32, diff --git a/ryujinx/src/Ryujinx.Graphics.OpenGL/Pipeline.cs b/ryujinx/src/Ryujinx.Graphics.OpenGL/Pipeline.cs index 6d066bb676..54f6b3f7b2 100644 --- a/ryujinx/src/Ryujinx.Graphics.OpenGL/Pipeline.cs +++ b/ryujinx/src/Ryujinx.Graphics.OpenGL/Pipeline.cs @@ -963,6 +963,11 @@ namespace Ryujinx.Graphics.OpenGL (array as ImageArray).Bind(binding); } + public void SetImageArraySeparate(ShaderStage stage, int setIndex, IImageArray array) + { + throw new NotSupportedException("OpenGL does not support descriptor sets."); + } + public void SetIndexBuffer(BufferRange buffer, IndexType type) { _elementsType = type.Convert(); @@ -1312,6 +1317,11 @@ namespace Ryujinx.Graphics.OpenGL (array as TextureArray).Bind(binding); } + public void SetTextureArraySeparate(ShaderStage stage, int setIndex, ITextureArray array) + { + throw new NotSupportedException("OpenGL does not support descriptor sets."); + } + public void SetTransformFeedbackBuffers(ReadOnlySpan buffers) { if (_tfEnabled) diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/BufferDescriptor.cs b/ryujinx/src/Ryujinx.Graphics.Shader/BufferDescriptor.cs index ead1c5e67b..11d4e3c116 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/BufferDescriptor.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/BufferDescriptor.cs @@ -4,14 +4,16 @@ namespace Ryujinx.Graphics.Shader { // New fields should be added to the end of the struct to keep disk shader cache compatibility. + public readonly int Set; public readonly int Binding; public readonly byte Slot; public readonly byte SbCbSlot; public readonly ushort SbCbOffset; public readonly BufferUsageFlags Flags; - public BufferDescriptor(int binding, int slot) + public BufferDescriptor(int set, int binding, int slot) { + Set = set; Binding = binding; Slot = (byte)slot; SbCbSlot = 0; @@ -19,8 +21,9 @@ namespace Ryujinx.Graphics.Shader Flags = BufferUsageFlags.None; } - public BufferDescriptor(int binding, int slot, int sbCbSlot, int sbCbOffset, BufferUsageFlags flags) + public BufferDescriptor(int set, int binding, int slot, int sbCbSlot, int sbCbOffset, BufferUsageFlags flags) { + Set = set; Binding = binding; Slot = (byte)slot; SbCbSlot = (byte)sbCbSlot; diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs b/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs index f0e57b534b..4308b08f81 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Glsl/Instructions/InstGenMemory.cs @@ -462,7 +462,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions } else { - context.Properties.Textures.TryGetValue(texOp.Binding, out TextureDefinition definition); + context.Properties.Textures.TryGetValue(texOp.GetTextureSetAndBinding(), out TextureDefinition definition); bool hasLod = !definition.Type.HasFlag(SamplerType.Multisample) && (definition.Type & SamplerType.Mask) != SamplerType.TextureBuffer; string texCall; @@ -639,7 +639,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions private static string GetSamplerName(CodeGenContext context, AstTextureOperation texOp, ref int srcIndex) { - TextureDefinition textureDefinition = context.Properties.Textures[texOp.Binding]; + TextureDefinition textureDefinition = context.Properties.Textures[texOp.GetTextureSetAndBinding()]; string name = textureDefinition.Name; if (textureDefinition.ArrayLength != 1) @@ -649,7 +649,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions if (texOp.IsSeparate) { - TextureDefinition samplerDefinition = context.Properties.Textures[texOp.SamplerBinding]; + TextureDefinition samplerDefinition = context.Properties.Textures[texOp.GetSamplerSetAndBinding()]; string samplerName = samplerDefinition.Name; if (samplerDefinition.ArrayLength != 1) @@ -665,7 +665,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Glsl.Instructions private static string GetImageName(CodeGenContext context, AstTextureOperation texOp, ref int srcIndex) { - TextureDefinition definition = context.Properties.Images[texOp.Binding]; + TextureDefinition definition = context.Properties.Images[texOp.GetTextureSetAndBinding()]; string name = definition.Name; if (definition.ArrayLength != 1) diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs b/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs index 2b1fdf44c3..f3be29bb9d 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/CodeGenContext.cs @@ -33,9 +33,9 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv public Dictionary LocalMemories { get; } = new(); public Dictionary SharedMemories { get; } = new(); - public Dictionary SamplersTypes { get; } = new(); - public Dictionary Samplers { get; } = new(); - public Dictionary Images { get; } = new(); + public Dictionary SamplersTypes { get; } = new(); + public Dictionary Samplers { get; } = new(); + public Dictionary Images { get; } = new(); public Dictionary Inputs { get; } = new(); public Dictionary Outputs { get; } = new(); diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs b/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs index 37df4df802..55d35bf0d1 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Declarations.cs @@ -208,13 +208,13 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv var sampledImageVariable = context.Variable(sampledImageArrayPointerType, StorageClass.UniformConstant); - context.Samplers.Add(sampler.Binding, new SamplerDeclaration( + context.Samplers.Add(new(sampler.Set, sampler.Binding), new SamplerDeclaration( imageType, sampledImageType, sampledImagePointerType, sampledImageVariable, sampler.ArrayLength != 1)); - context.SamplersTypes.Add(sampler.Binding, sampler.Type); + context.SamplersTypes.Add(new(sampler.Set, sampler.Binding), sampler.Type); context.Name(sampledImageVariable, sampler.Name); context.Decorate(sampledImageVariable, Decoration.DescriptorSet, (LiteralInteger)setIndex); @@ -256,7 +256,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv var imageVariable = context.Variable(imageArrayPointerType, StorageClass.UniformConstant); - context.Images.Add(image.Binding, new ImageDeclaration(imageType, imagePointerType, imageVariable, image.ArrayLength != 1)); + context.Images.Add(new(image.Set, image.Binding), new ImageDeclaration(imageType, imagePointerType, imageVariable, image.ArrayLength != 1)); context.Name(imageVariable, image.Name); context.Decorate(imageVariable, Decoration.DescriptorSet, (LiteralInteger)setIndex); diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs b/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs index 34f8532a65..6206985d85 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/CodeGen/Spirv/Instructions.cs @@ -602,7 +602,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv return context.Get(type, texOp.GetSource(srcIndex++)); } - ImageDeclaration declaration = context.Images[texOp.Binding]; + ImageDeclaration declaration = context.Images[texOp.GetTextureSetAndBinding()]; SpvInstruction image = declaration.Image; SpvInstruction resultType = context.GetType(componentType); @@ -681,7 +681,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv return context.Get(type, texOp.GetSource(srcIndex++)); } - ImageDeclaration declaration = context.Images[texOp.Binding]; + ImageDeclaration declaration = context.Images[texOp.GetTextureSetAndBinding()]; SpvInstruction image = declaration.Image; if (declaration.IsIndexed) @@ -738,7 +738,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv return context.Get(type, texOp.GetSource(srcIndex++)); } - ImageDeclaration declaration = context.Images[texOp.Binding]; + ImageDeclaration declaration = context.Images[texOp.GetTextureSetAndBinding()]; SpvInstruction image = declaration.Image; if (declaration.IsIndexed) @@ -837,7 +837,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv return context.Get(type, texOp.GetSource(srcIndex++)); } - SamplerDeclaration declaration = context.Samplers[texOp.Binding]; + SamplerDeclaration declaration = context.Samplers[texOp.GetTextureSetAndBinding()]; SpvInstruction image = GenerateSampledImageLoad(context, texOp, declaration, ref srcIndex); int pCount = texOp.Type.GetDimensions(); @@ -1161,7 +1161,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv return context.Get(type, texOp.GetSource(srcIndex++)); } - SamplerDeclaration declaration = context.Samplers[texOp.Binding]; + SamplerDeclaration declaration = context.Samplers[texOp.GetTextureSetAndBinding()]; SpvInstruction image = GenerateSampledImageLoad(context, texOp, declaration, ref srcIndex); int coordsCount = texOp.Type.GetDimensions(); @@ -1433,7 +1433,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv int srcIndex = 0; - SamplerDeclaration declaration = context.Samplers[texOp.Binding]; + SamplerDeclaration declaration = context.Samplers[texOp.GetTextureSetAndBinding()]; SpvInstruction image = GenerateSampledImageLoad(context, texOp, declaration, ref srcIndex); image = context.Image(declaration.ImageType, image); @@ -1449,7 +1449,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv int srcIndex = 0; - SamplerDeclaration declaration = context.Samplers[texOp.Binding]; + SamplerDeclaration declaration = context.Samplers[texOp.GetTextureSetAndBinding()]; SpvInstruction image = GenerateSampledImageLoad(context, texOp, declaration, ref srcIndex); image = context.Image(declaration.ImageType, image); @@ -1460,7 +1460,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv } else { - var type = context.SamplersTypes[texOp.Binding]; + var type = context.SamplersTypes[texOp.GetTextureSetAndBinding()]; bool hasLod = !type.HasFlag(SamplerType.Multisample) && type != SamplerType.TextureBuffer; int dimensions = (type & SamplerType.Mask) == SamplerType.TextureCube ? 2 : type.GetDimensions(); @@ -1889,7 +1889,7 @@ namespace Ryujinx.Graphics.Shader.CodeGen.Spirv { image = context.Load(declaration.ImageType, image); - SamplerDeclaration samplerDeclaration = context.Samplers[texOp.SamplerBinding]; + SamplerDeclaration samplerDeclaration = context.Samplers[texOp.GetSamplerSetAndBinding()]; SpvInstruction sampler = samplerDeclaration.Image; diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs b/ryujinx/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs index 3dc4ad907b..4e6d6edf9b 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/IGpuAccessor.cs @@ -27,34 +27,43 @@ namespace Ryujinx.Graphics.Shader ReadOnlySpan GetCode(ulong address, int minimumSize); /// - /// Queries the binding number of a constant buffer. + /// Gets the binding number of a constant buffer. /// /// Constant buffer index /// Binding number - int CreateConstantBufferBinding(int index); + SetBindingPair CreateConstantBufferBinding(int index); /// - /// Queries the binding number of an image. + /// Gets the binding number of an image. /// /// For array of images, the number of elements of the array, otherwise it should be 1 /// Indicates if the image is a buffer image /// Binding number - int CreateImageBinding(int count, bool isBuffer); + SetBindingPair CreateImageBinding(int count, bool isBuffer); /// - /// Queries the binding number of a storage buffer. + /// Gets the binding number of a storage buffer. /// /// Storage buffer index /// Binding number - int CreateStorageBufferBinding(int index); + SetBindingPair CreateStorageBufferBinding(int index); /// - /// Queries the binding number of a texture. + /// Gets the binding number of a texture. /// /// For array of textures, the number of elements of the array, otherwise it should be 1 /// Indicates if the texture is a buffer texture /// Binding number - int CreateTextureBinding(int count, bool isBuffer); + SetBindingPair CreateTextureBinding(int count, bool isBuffer); + + /// + /// Gets the set index for an additional set, or -1 if there's no extra set available. + /// + /// Extra set index, or -1 if not available + int CreateExtraSet() + { + return -1; + } /// /// Queries Local Size X for compute shaders. diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitSurface.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitSurface.cs index 0aac0ffa8c..383e82c697 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitSurface.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitSurface.cs @@ -278,7 +278,7 @@ namespace Ryujinx.Graphics.Shader.Instructions flags |= TextureFlags.Bindless; } - int binding = isBindless ? 0 : context.ResourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.ImageAtomic, type, format, @@ -286,7 +286,7 @@ namespace Ryujinx.Graphics.Shader.Instructions TextureOperation.DefaultCbufSlot, imm); - Operand res = context.ImageAtomic(type, format, flags, binding, sources); + Operand res = context.ImageAtomic(type, format, flags, setAndBinding, sources); context.Copy(d, res); } @@ -389,7 +389,7 @@ namespace Ryujinx.Graphics.Shader.Instructions TextureFormat format = isBindless ? TextureFormat.Unknown : ShaderProperties.GetTextureFormat(context.TranslatorContext.GpuAccessor, handle); - int binding = isBindless ? 0 : context.ResourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.ImageLoad, type, format, @@ -397,7 +397,7 @@ namespace Ryujinx.Graphics.Shader.Instructions TextureOperation.DefaultCbufSlot, handle); - context.ImageLoad(type, format, flags, binding, (int)componentMask, dests, sources); + context.ImageLoad(type, format, flags, setAndBinding, (int)componentMask, dests, sources); } else { @@ -432,7 +432,7 @@ namespace Ryujinx.Graphics.Shader.Instructions TextureFormat format = GetTextureFormat(size); - int binding = isBindless ? 0 : context.ResourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.ImageLoad, type, format, @@ -440,7 +440,7 @@ namespace Ryujinx.Graphics.Shader.Instructions TextureOperation.DefaultCbufSlot, handle); - context.ImageLoad(type, format, flags, binding, compMask, dests, sources); + context.ImageLoad(type, format, flags, setAndBinding, compMask, dests, sources); switch (size) { @@ -552,7 +552,7 @@ namespace Ryujinx.Graphics.Shader.Instructions flags |= TextureFlags.Bindless; } - int binding = isBindless ? 0 : context.ResourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.ImageAtomic, type, format, @@ -560,7 +560,7 @@ namespace Ryujinx.Graphics.Shader.Instructions TextureOperation.DefaultCbufSlot, imm); - context.ImageAtomic(type, format, flags, binding, sources); + context.ImageAtomic(type, format, flags, setAndBinding, sources); } private static void EmitSust( @@ -679,7 +679,7 @@ namespace Ryujinx.Graphics.Shader.Instructions flags |= TextureFlags.Coherent; } - int binding = isBindless ? 0 : context.ResourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.ImageStore, type, format, @@ -687,7 +687,7 @@ namespace Ryujinx.Graphics.Shader.Instructions TextureOperation.DefaultCbufSlot, handle); - context.ImageStore(type, format, flags, binding, sources); + context.ImageStore(type, format, flags, setAndBinding, sources); } private static int GetComponentSizeInBytesLog2(SuatomSize size) diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitTexture.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitTexture.cs index 06daa26a0a..2076262daf 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitTexture.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Instructions/InstEmitTexture.cs @@ -885,7 +885,7 @@ namespace Ryujinx.Graphics.Shader.Instructions return Register(dest++, RegisterType.Gpr); } - int binding = isBindless ? 0 : context.ResourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.Lod, type, TextureFormat.Unknown, @@ -913,7 +913,7 @@ namespace Ryujinx.Graphics.Shader.Instructions else { // The instruction component order is the inverse of GLSL's. - Operand res = context.Lod(type, flags, binding, compIndex ^ 1, sources); + Operand res = context.Lod(type, flags, setAndBinding, compIndex ^ 1, sources); res = context.FPMultiply(res, ConstF(256.0f)); @@ -1116,12 +1116,12 @@ namespace Ryujinx.Graphics.Shader.Instructions } TextureFlags flags = isBindless ? TextureFlags.Bindless : TextureFlags.None; - int binding; + SetBindingPair setAndBinding; switch (query) { case TexQuery.TexHeaderDimension: - binding = isBindless ? 0 : context.ResourceManager.GetTextureOrImageBinding( + setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.TextureQuerySize, type, TextureFormat.Unknown, @@ -1140,13 +1140,13 @@ namespace Ryujinx.Graphics.Shader.Instructions break; } - context.Copy(d, context.TextureQuerySize(type, flags, binding, compIndex, sources)); + context.Copy(d, context.TextureQuerySize(type, flags, setAndBinding, compIndex, sources)); } } break; case TexQuery.TexHeaderTextureType: - binding = isBindless ? 0 : context.ResourceManager.GetTextureOrImageBinding( + setAndBinding = isBindless ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.TextureQuerySamples, type, TextureFormat.Unknown, @@ -1171,7 +1171,7 @@ namespace Ryujinx.Graphics.Shader.Instructions if (d != null) { - context.Copy(d, context.TextureQuerySamples(type, flags, binding, sources)); + context.Copy(d, context.TextureQuerySamples(type, flags, setAndBinding, sources)); } } break; @@ -1191,7 +1191,7 @@ namespace Ryujinx.Graphics.Shader.Instructions Operand[] dests, Operand[] sources) { - int binding = flags.HasFlag(TextureFlags.Bindless) ? 0 : context.ResourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = flags.HasFlag(TextureFlags.Bindless) ? default : context.ResourceManager.GetTextureOrImageBinding( Instruction.TextureSample, type, TextureFormat.Unknown, @@ -1199,7 +1199,7 @@ namespace Ryujinx.Graphics.Shader.Instructions TextureOperation.DefaultCbufSlot, handle); - context.TextureSample(type, flags, binding, componentMask, dests, sources); + context.TextureSample(type, flags, setAndBinding, componentMask, dests, sources); } private static SamplerType ConvertSamplerType(TexDim dimensions) diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/TextureOperation.cs b/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/TextureOperation.cs index 74ec5ca611..7eee8f2e9c 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/TextureOperation.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/IntermediateRepresentation/TextureOperation.cs @@ -8,7 +8,9 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation public TextureFormat Format { get; set; } public TextureFlags Flags { get; private set; } + public int Set { get; private set; } public int Binding { get; private set; } + public int SamplerSet { get; private set; } public int SamplerBinding { get; private set; } public TextureOperation( @@ -16,6 +18,7 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation SamplerType type, TextureFormat format, TextureFlags flags, + int set, int binding, int compIndex, Operand[] dests, @@ -24,24 +27,28 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation Type = type; Format = format; Flags = flags; + Set = set; Binding = binding; + SamplerSet = -1; SamplerBinding = -1; } - public void TurnIntoArray(int binding) + public void TurnIntoArray(SetBindingPair setAndBinding) { Flags &= ~TextureFlags.Bindless; - Binding = binding; + Set = setAndBinding.SetIndex; + Binding = setAndBinding.Binding; } - public void TurnIntoArray(int textureBinding, int samplerBinding) + public void TurnIntoArray(SetBindingPair textureSetAndBinding, SetBindingPair samplerSetAndBinding) { - TurnIntoArray(textureBinding); + TurnIntoArray(textureSetAndBinding); - SamplerBinding = samplerBinding; + SamplerSet = samplerSetAndBinding.SetIndex; + SamplerBinding = samplerSetAndBinding.Binding; } - public void SetBinding(int binding) + public void SetBinding(SetBindingPair setAndBinding) { if ((Flags & TextureFlags.Bindless) != 0) { @@ -50,7 +57,8 @@ namespace Ryujinx.Graphics.Shader.IntermediateRepresentation RemoveSource(0); } - Binding = binding; + Set = setAndBinding.SetIndex; + Binding = setAndBinding.Binding; } public void SetLodLevelFlag() diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/SetBindingPair.cs b/ryujinx/src/Ryujinx.Graphics.Shader/SetBindingPair.cs new file mode 100644 index 0000000000..1e8a4f9c61 --- /dev/null +++ b/ryujinx/src/Ryujinx.Graphics.Shader/SetBindingPair.cs @@ -0,0 +1,4 @@ +namespace Ryujinx.Graphics.Shader +{ + public readonly record struct SetBindingPair(int SetIndex, int Binding); +} diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/AstTextureOperation.cs b/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/AstTextureOperation.cs index 4068c41271..867cae8536 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/AstTextureOperation.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/AstTextureOperation.cs @@ -8,7 +8,9 @@ namespace Ryujinx.Graphics.Shader.StructuredIr public TextureFormat Format { get; } public TextureFlags Flags { get; } + public int Set { get; } public int Binding { get; } + public int SamplerSet { get; } public int SamplerBinding { get; } public bool IsSeparate => SamplerBinding >= 0; @@ -18,7 +20,9 @@ namespace Ryujinx.Graphics.Shader.StructuredIr SamplerType type, TextureFormat format, TextureFlags flags, + int set, int binding, + int samplerSet, int samplerBinding, int index, params IAstNode[] sources) : base(inst, StorageKind.None, false, index, sources, sources.Length) @@ -26,8 +30,20 @@ namespace Ryujinx.Graphics.Shader.StructuredIr Type = type; Format = format; Flags = flags; + Set = set; Binding = binding; + SamplerSet = samplerSet; SamplerBinding = samplerBinding; } + + public SetBindingPair GetTextureSetAndBinding() + { + return new SetBindingPair(Set, Binding); + } + + public SetBindingPair GetSamplerSetAndBinding() + { + return new SetBindingPair(SamplerSet, SamplerBinding); + } } } diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs b/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs index 8c12c2aaf5..53ed6bfccd 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/ShaderProperties.cs @@ -6,15 +6,15 @@ namespace Ryujinx.Graphics.Shader.StructuredIr { private readonly Dictionary _constantBuffers; private readonly Dictionary _storageBuffers; - private readonly Dictionary _textures; - private readonly Dictionary _images; + private readonly Dictionary _textures; + private readonly Dictionary _images; private readonly Dictionary _localMemories; private readonly Dictionary _sharedMemories; public IReadOnlyDictionary ConstantBuffers => _constantBuffers; public IReadOnlyDictionary StorageBuffers => _storageBuffers; - public IReadOnlyDictionary Textures => _textures; - public IReadOnlyDictionary Images => _images; + public IReadOnlyDictionary Textures => _textures; + public IReadOnlyDictionary Images => _images; public IReadOnlyDictionary LocalMemories => _localMemories; public IReadOnlyDictionary SharedMemories => _sharedMemories; @@ -22,8 +22,8 @@ namespace Ryujinx.Graphics.Shader.StructuredIr { _constantBuffers = new Dictionary(); _storageBuffers = new Dictionary(); - _textures = new Dictionary(); - _images = new Dictionary(); + _textures = new Dictionary(); + _images = new Dictionary(); _localMemories = new Dictionary(); _sharedMemories = new Dictionary(); } @@ -40,12 +40,12 @@ namespace Ryujinx.Graphics.Shader.StructuredIr public void AddOrUpdateTexture(TextureDefinition definition) { - _textures[definition.Binding] = definition; + _textures[new(definition.Set, definition.Binding)] = definition; } public void AddOrUpdateImage(TextureDefinition definition) { - _images[definition.Binding] = definition; + _images[new(definition.Set, definition.Binding)] = definition; } public int AddLocalMemory(MemoryDefinition definition) diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs b/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs index c4ebaee73f..88053658dd 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/StructuredIr/StructuredProgram.cs @@ -169,7 +169,17 @@ namespace Ryujinx.Graphics.Shader.StructuredIr AstTextureOperation GetAstTextureOperation(TextureOperation texOp) { - return new AstTextureOperation(inst, texOp.Type, texOp.Format, texOp.Flags, texOp.Binding, texOp.SamplerBinding, texOp.Index, sources); + return new AstTextureOperation( + inst, + texOp.Type, + texOp.Format, + texOp.Flags, + texOp.Set, + texOp.Binding, + texOp.SamplerSet, + texOp.SamplerBinding, + texOp.Index, + sources); } int componentsCount = BitOperations.PopCount((uint)operation.Index); diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/TextureDescriptor.cs b/ryujinx/src/Ryujinx.Graphics.Shader/TextureDescriptor.cs index d287a1aa7b..1e387407d8 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/TextureDescriptor.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/TextureDescriptor.cs @@ -4,6 +4,7 @@ namespace Ryujinx.Graphics.Shader { // New fields should be added to the end of the struct to keep disk shader cache compatibility. + public readonly int Set; public readonly int Binding; public readonly SamplerType Type; @@ -18,6 +19,7 @@ namespace Ryujinx.Graphics.Shader public readonly TextureUsageFlags Flags; public TextureDescriptor( + int set, int binding, SamplerType type, TextureFormat format, @@ -27,6 +29,7 @@ namespace Ryujinx.Graphics.Shader bool separate, TextureUsageFlags flags) { + Set = set; Binding = binding; Type = type; Format = format; diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs index e1157eea41..5e07b39f12 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContext.cs @@ -124,7 +124,7 @@ namespace Ryujinx.Graphics.Shader.Translation this.TextureSample( SamplerType.TextureBuffer, TextureFlags.IntCoords, - ResourceManager.Reservations.IndexBufferTextureBinding, + ResourceManager.Reservations.GetIndexBufferTextureSetAndBinding(), 1, new[] { vertexIndexVr }, new[] { this.IAdd(ibBaseOffset, outputVertexOffset) }); @@ -145,7 +145,7 @@ namespace Ryujinx.Graphics.Shader.Translation this.TextureSample( SamplerType.TextureBuffer, TextureFlags.IntCoords, - ResourceManager.Reservations.TopologyRemapBufferTextureBinding, + ResourceManager.Reservations.GetTopologyRemapBufferTextureSetAndBinding(), 1, new[] { vertexIndex }, new[] { this.IAdd(baseVertex, Const(index)) }); diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContextInsts.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContextInsts.cs index 9e314c6204..5bdbb0025a 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContextInsts.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/EmitterContextInsts.cs @@ -618,12 +618,21 @@ namespace Ryujinx.Graphics.Shader.Translation SamplerType type, TextureFormat format, TextureFlags flags, - int binding, + SetBindingPair setAndBinding, Operand[] sources) { Operand dest = Local(); - context.Add(new TextureOperation(Instruction.ImageAtomic, type, format, flags, binding, 0, new[] { dest }, sources)); + context.Add(new TextureOperation( + Instruction.ImageAtomic, + type, + format, + flags, + setAndBinding.SetIndex, + setAndBinding.Binding, + 0, + new[] { dest }, + sources)); return dest; } @@ -633,12 +642,21 @@ namespace Ryujinx.Graphics.Shader.Translation SamplerType type, TextureFormat format, TextureFlags flags, - int binding, + SetBindingPair setAndBinding, int compMask, Operand[] dests, Operand[] sources) { - context.Add(new TextureOperation(Instruction.ImageLoad, type, format, flags, binding, compMask, dests, sources)); + context.Add(new TextureOperation( + Instruction.ImageLoad, + type, + format, + flags, + setAndBinding.SetIndex, + setAndBinding.Binding, + compMask, + dests, + sources)); } public static void ImageStore( @@ -646,10 +664,19 @@ namespace Ryujinx.Graphics.Shader.Translation SamplerType type, TextureFormat format, TextureFlags flags, - int binding, + SetBindingPair setAndBinding, Operand[] sources) { - context.Add(new TextureOperation(Instruction.ImageStore, type, format, flags, binding, 0, null, sources)); + context.Add(new TextureOperation( + Instruction.ImageStore, + type, + format, + flags, + setAndBinding.SetIndex, + setAndBinding.Binding, + 0, + null, + sources)); } public static Operand IsNan(this EmitterContext context, Operand a, Instruction fpType = Instruction.FP32) @@ -718,13 +745,22 @@ namespace Ryujinx.Graphics.Shader.Translation this EmitterContext context, SamplerType type, TextureFlags flags, - int binding, + SetBindingPair setAndBinding, int compIndex, Operand[] sources) { Operand dest = Local(); - context.Add(new TextureOperation(Instruction.Lod, type, TextureFormat.Unknown, flags, binding, compIndex, new[] { dest }, sources)); + context.Add(new TextureOperation( + Instruction.Lod, + type, + TextureFormat.Unknown, + flags, + setAndBinding.SetIndex, + setAndBinding.Binding, + compIndex, + new[] { dest }, + sources)); return dest; } @@ -889,24 +925,42 @@ namespace Ryujinx.Graphics.Shader.Translation this EmitterContext context, SamplerType type, TextureFlags flags, - int binding, + SetBindingPair setAndBinding, int compMask, Operand[] dests, Operand[] sources) { - context.Add(new TextureOperation(Instruction.TextureSample, type, TextureFormat.Unknown, flags, binding, compMask, dests, sources)); + context.Add(new TextureOperation( + Instruction.TextureSample, + type, + TextureFormat.Unknown, + flags, + setAndBinding.SetIndex, + setAndBinding.Binding, + compMask, + dests, + sources)); } public static Operand TextureQuerySamples( this EmitterContext context, SamplerType type, TextureFlags flags, - int binding, + SetBindingPair setAndBinding, Operand[] sources) { Operand dest = Local(); - context.Add(new TextureOperation(Instruction.TextureQuerySamples, type, TextureFormat.Unknown, flags, binding, 0, new[] { dest }, sources)); + context.Add(new TextureOperation( + Instruction.TextureQuerySamples, + type, + TextureFormat.Unknown, + flags, + setAndBinding.SetIndex, + setAndBinding.Binding, + 0, + new[] { dest }, + sources)); return dest; } @@ -915,13 +969,22 @@ namespace Ryujinx.Graphics.Shader.Translation this EmitterContext context, SamplerType type, TextureFlags flags, - int binding, + SetBindingPair setAndBinding, int compIndex, Operand[] sources) { Operand dest = Local(); - context.Add(new TextureOperation(Instruction.TextureQuerySize, type, TextureFormat.Unknown, flags, binding, compIndex, new[] { dest }, sources)); + context.Add(new TextureOperation( + Instruction.TextureQuerySize, + type, + TextureFormat.Unknown, + flags, + setAndBinding.SetIndex, + setAndBinding.Binding, + compIndex, + new[] { dest }, + sources)); return dest; } diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs index 4128af241b..02a83fbe40 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessElimination.cs @@ -38,6 +38,12 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations // If we can't do bindless elimination, remove the texture operation. // Set any destination variables to zero. + string typeName = texOp.Inst.IsImage() + ? texOp.Type.ToGlslImageType(texOp.Format.GetComponentType()) + : texOp.Type.ToGlslTextureType(); + + gpuAccessor.Log($"Failed to find handle source for bindless access of type \"{typeName}\"."); + for (int destIndex = 0; destIndex < texOp.DestsCount; destIndex++) { block.Operations.AddBefore(node, new Operation(Instruction.Copy, texOp.GetDest(destIndex), OperandHelper.Const(0))); @@ -62,17 +68,22 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations return false; } - Operand nvHandle = texOp.GetSource(0); + Operand bindlessHandle = texOp.GetSource(0); - if (nvHandle.AsgOp is not Operation handleOp || - handleOp.Inst != Instruction.Load || - (handleOp.StorageKind != StorageKind.Input && handleOp.StorageKind != StorageKind.StorageBuffer)) + if (bindlessHandle.AsgOp is PhiNode phi) { - // Right now, we only allow bindless access when the handle comes from a shader input or storage buffer. - // This is an artificial limitation to prevent it from being used in cases where it - // would have a large performance impact of loading all textures in the pool. - // It might be removed in the future, if we can mitigate the performance impact. + for (int srcIndex = 0; srcIndex < phi.SourcesCount; srcIndex++) + { + Operand phiSource = phi.GetSource(srcIndex); + if (phiSource.AsgOp is not PhiNode && !IsBindlessAccessAllowed(phiSource)) + { + return false; + } + } + } + else if (!IsBindlessAccessAllowed(bindlessHandle)) + { return false; } @@ -80,8 +91,8 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations Operand samplerHandle = OperandHelper.Local(); Operand textureIndex = OperandHelper.Local(); - block.Operations.AddBefore(node, new Operation(Instruction.BitwiseAnd, textureHandle, nvHandle, OperandHelper.Const(0xfffff))); - block.Operations.AddBefore(node, new Operation(Instruction.ShiftRightU32, samplerHandle, nvHandle, OperandHelper.Const(20))); + block.Operations.AddBefore(node, new Operation(Instruction.BitwiseAnd, textureHandle, bindlessHandle, OperandHelper.Const(0xfffff))); + block.Operations.AddBefore(node, new Operation(Instruction.ShiftRightU32, samplerHandle, bindlessHandle, OperandHelper.Const(20))); int texturePoolLength = Math.Max(BindlessToArray.MinimumArrayLength, gpuAccessor.QueryTextureArrayLengthFromPool()); @@ -91,7 +102,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations bool hasSampler = !texOp.Inst.IsImage(); - int textureBinding = resourceManager.GetTextureOrImageBinding( + SetBindingPair textureSetAndBinding = resourceManager.GetTextureOrImageBinding( texOp.Inst, texOp.Type, texOp.Format, @@ -111,7 +122,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations texOp.InsertSource(1, samplerIndex); - int samplerBinding = resourceManager.GetTextureOrImageBinding( + SetBindingPair samplerSetAndBinding = resourceManager.GetTextureOrImageBinding( texOp.Inst, SamplerType.None, texOp.Format, @@ -120,11 +131,35 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations TextureHandle.PackOffsets(0, 0, TextureHandleType.Direct), samplerPoolLength); - texOp.TurnIntoArray(textureBinding, samplerBinding); + texOp.TurnIntoArray(textureSetAndBinding, samplerSetAndBinding); } else { - texOp.TurnIntoArray(textureBinding); + texOp.TurnIntoArray(textureSetAndBinding); + } + + return true; + } + + private static bool IsBindlessAccessAllowed(Operand nvHandle) + { + if (nvHandle.Type == OperandType.ConstantBuffer) + { + // Bindless access with handles from constant buffer is allowed. + + return true; + } + + if (nvHandle.AsgOp is not Operation handleOp || + handleOp.Inst != Instruction.Load || + (handleOp.StorageKind != StorageKind.Input && handleOp.StorageKind != StorageKind.StorageBuffer)) + { + // Right now, we only allow bindless access when the handle comes from a shader input or storage buffer. + // This is an artificial limitation to prevent it from being used in cases where it + // would have a large performance impact of loading all textures in the pool. + // It might be removed in the future, if we can mitigate the performance impact. + + return false; } return true; @@ -445,7 +480,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations } } - int binding = resourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = resourceManager.GetTextureOrImageBinding( texOp.Inst, texOp.Type, texOp.Format, @@ -453,7 +488,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations cbufSlot, cbufOffset); - texOp.SetBinding(binding); + texOp.SetBinding(setAndBinding); } } } diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessToArray.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessToArray.cs index f2be7975da..8eed139d6c 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessToArray.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Optimizations/BindlessToArray.cs @@ -221,7 +221,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations private static void TurnIntoArray(ResourceManager resourceManager, TextureOperation texOp, int cbufSlot, int handleIndex, int length) { - int binding = resourceManager.GetTextureOrImageBinding( + SetBindingPair setAndBinding = resourceManager.GetTextureOrImageBinding( texOp.Inst, texOp.Type, texOp.Format, @@ -230,7 +230,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Optimizations handleIndex, length); - texOp.TurnIntoArray(binding); + texOp.TurnIntoArray(setAndBinding); } } } diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs index 890501c919..94691a5b4c 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceManager.cs @@ -20,8 +20,8 @@ namespace Ryujinx.Graphics.Shader.Translation private readonly ShaderStage _stage; private readonly string _stagePrefix; - private readonly int[] _cbSlotToBindingMap; - private readonly int[] _sbSlotToBindingMap; + private readonly SetBindingPair[] _cbSlotToBindingMap; + private readonly SetBindingPair[] _sbSlotToBindingMap; private uint _sbSlotWritten; private readonly Dictionary _sbSlots; @@ -33,6 +33,7 @@ namespace Ryujinx.Graphics.Shader.Translation private struct TextureMeta { + public int Set; public int Binding; public bool AccurateType; public SamplerType Type; @@ -64,10 +65,10 @@ namespace Ryujinx.Graphics.Shader.Translation _stage = stage; _stagePrefix = GetShaderStagePrefix(stage); - _cbSlotToBindingMap = new int[18]; - _sbSlotToBindingMap = new int[16]; - _cbSlotToBindingMap.AsSpan().Fill(-1); - _sbSlotToBindingMap.AsSpan().Fill(-1); + _cbSlotToBindingMap = new SetBindingPair[18]; + _sbSlotToBindingMap = new SetBindingPair[16]; + _cbSlotToBindingMap.AsSpan().Fill(new(-1, -1)); + _sbSlotToBindingMap.AsSpan().Fill(new(-1, -1)); _sbSlots = new(); _sbSlotsReverse = new(); @@ -146,16 +147,16 @@ namespace Ryujinx.Graphics.Shader.Translation public int GetConstantBufferBinding(int slot) { - int binding = _cbSlotToBindingMap[slot]; - if (binding < 0) + SetBindingPair setAndBinding = _cbSlotToBindingMap[slot]; + if (setAndBinding.Binding < 0) { - binding = _gpuAccessor.CreateConstantBufferBinding(slot); - _cbSlotToBindingMap[slot] = binding; + setAndBinding = _gpuAccessor.CreateConstantBufferBinding(slot); + _cbSlotToBindingMap[slot] = setAndBinding; string slotNumber = slot.ToString(CultureInfo.InvariantCulture); - AddNewConstantBuffer(binding, $"{_stagePrefix}_c{slotNumber}"); + AddNewConstantBuffer(setAndBinding.SetIndex, setAndBinding.Binding, $"{_stagePrefix}_c{slotNumber}"); } - return binding; + return setAndBinding.Binding; } public bool TryGetStorageBufferBinding(int sbCbSlot, int sbCbOffset, bool write, out int binding) @@ -166,14 +167,14 @@ namespace Ryujinx.Graphics.Shader.Translation return false; } - binding = _sbSlotToBindingMap[slot]; + SetBindingPair setAndBinding = _sbSlotToBindingMap[slot]; - if (binding < 0) + if (setAndBinding.Binding < 0) { - binding = _gpuAccessor.CreateStorageBufferBinding(slot); - _sbSlotToBindingMap[slot] = binding; + setAndBinding = _gpuAccessor.CreateStorageBufferBinding(slot); + _sbSlotToBindingMap[slot] = setAndBinding; string slotNumber = slot.ToString(CultureInfo.InvariantCulture); - AddNewStorageBuffer(binding, $"{_stagePrefix}_s{slotNumber}"); + AddNewStorageBuffer(setAndBinding.SetIndex, setAndBinding.Binding, $"{_stagePrefix}_s{slotNumber}"); } if (write) @@ -181,6 +182,7 @@ namespace Ryujinx.Graphics.Shader.Translation _sbSlotWritten |= 1u << slot; } + binding = setAndBinding.Binding; return true; } @@ -208,7 +210,7 @@ namespace Ryujinx.Graphics.Shader.Translation { for (slot = 0; slot < _cbSlotToBindingMap.Length; slot++) { - if (_cbSlotToBindingMap[slot] == binding) + if (_cbSlotToBindingMap[slot].Binding == binding) { return true; } @@ -218,7 +220,7 @@ namespace Ryujinx.Graphics.Shader.Translation return false; } - public int GetTextureOrImageBinding( + public SetBindingPair GetTextureOrImageBinding( Instruction inst, SamplerType type, TextureFormat format, @@ -240,7 +242,7 @@ namespace Ryujinx.Graphics.Shader.Translation format = TextureFormat.Unknown; } - int binding = GetTextureOrImageBinding( + SetBindingPair setAndBinding = GetTextureOrImageBinding( cbufSlot, handle, arrayLength, @@ -255,10 +257,10 @@ namespace Ryujinx.Graphics.Shader.Translation _gpuAccessor.RegisterTexture(handle, cbufSlot); - return binding; + return setAndBinding; } - private int GetTextureOrImageBinding( + private SetBindingPair GetTextureOrImageBinding( int cbufSlot, int handle, int arrayLength, @@ -311,21 +313,38 @@ namespace Ryujinx.Graphics.Shader.Translation UsageFlags = usageFlags, }; + int setIndex; int binding; if (dict.TryGetValue(info, out var existingMeta)) { dict[info] = MergeTextureMeta(meta, existingMeta); + setIndex = existingMeta.Set; binding = existingMeta.Binding; } else { - bool isBuffer = (type & SamplerType.Mask) == SamplerType.TextureBuffer; + if (arrayLength > 1 && (setIndex = _gpuAccessor.CreateExtraSet()) >= 0) + { + // We reserved an "extra set" for the array. + // In this case the binding is always the first one (0). + // Using separate sets for array is better as we need to do less descriptor set updates. - binding = isImage - ? _gpuAccessor.CreateImageBinding(arrayLength, isBuffer) - : _gpuAccessor.CreateTextureBinding(arrayLength, isBuffer); + binding = 0; + } + else + { + bool isBuffer = (type & SamplerType.Mask) == SamplerType.TextureBuffer; + SetBindingPair setAndBinding = isImage + ? _gpuAccessor.CreateImageBinding(arrayLength, isBuffer) + : _gpuAccessor.CreateTextureBinding(arrayLength, isBuffer); + + setIndex = setAndBinding.SetIndex; + binding = setAndBinding.Binding; + } + + meta.Set = setIndex; meta.Binding = binding; dict.Add(info, meta); @@ -355,7 +374,7 @@ namespace Ryujinx.Graphics.Shader.Translation } var definition = new TextureDefinition( - isImage ? 3 : 2, + setIndex, binding, arrayLength, separate, @@ -373,11 +392,12 @@ namespace Ryujinx.Graphics.Shader.Translation Properties.AddOrUpdateTexture(definition); } - return binding; + return new SetBindingPair(setIndex, binding); } private static TextureMeta MergeTextureMeta(TextureMeta meta, TextureMeta existingMeta) { + meta.Set = existingMeta.Set; meta.Binding = existingMeta.Binding; meta.UsageFlags |= existingMeta.UsageFlags; @@ -440,11 +460,11 @@ namespace Ryujinx.Graphics.Shader.Translation for (int slot = 0; slot < _cbSlotToBindingMap.Length; slot++) { - int binding = _cbSlotToBindingMap[slot]; + SetBindingPair setAndBinding = _cbSlotToBindingMap[slot]; - if (binding >= 0 && _usedConstantBufferBindings.Contains(binding)) + if (setAndBinding.Binding >= 0 && _usedConstantBufferBindings.Contains(setAndBinding.Binding)) { - descriptors[descriptorIndex++] = new BufferDescriptor(binding, slot); + descriptors[descriptorIndex++] = new BufferDescriptor(setAndBinding.SetIndex, setAndBinding.Binding, slot); } } @@ -464,13 +484,13 @@ namespace Ryujinx.Graphics.Shader.Translation foreach ((int key, int slot) in _sbSlots) { - int binding = _sbSlotToBindingMap[slot]; + SetBindingPair setAndBinding = _sbSlotToBindingMap[slot]; - if (binding >= 0) + if (setAndBinding.Binding >= 0) { (int sbCbSlot, int sbCbOffset) = UnpackSbCbInfo(key); BufferUsageFlags flags = (_sbSlotWritten & (1u << slot)) != 0 ? BufferUsageFlags.Write : BufferUsageFlags.None; - descriptors[descriptorIndex++] = new BufferDescriptor(binding, slot, sbCbSlot, sbCbOffset, flags); + descriptors[descriptorIndex++] = new BufferDescriptor(setAndBinding.SetIndex, setAndBinding.Binding, slot, sbCbSlot, sbCbOffset, flags); } } @@ -507,6 +527,7 @@ namespace Ryujinx.Graphics.Shader.Translation } descriptors.Add(new TextureDescriptor( + meta.Set, meta.Binding, meta.Type, info.Format, @@ -527,6 +548,7 @@ namespace Ryujinx.Graphics.Shader.Translation } descriptors.Add(new TextureDescriptor( + meta.Set, meta.Binding, meta.Type, info.Format, @@ -587,24 +609,24 @@ namespace Ryujinx.Graphics.Shader.Translation return false; } - private void AddNewConstantBuffer(int binding, string name) + private void AddNewConstantBuffer(int setIndex, int binding, string name) { StructureType type = new(new[] { new StructureField(AggregateType.Array | AggregateType.Vector4 | AggregateType.FP32, "data", Constants.ConstantBufferSize / 16), }); - Properties.AddOrUpdateConstantBuffer(new(BufferLayout.Std140, 0, binding, name, type)); + Properties.AddOrUpdateConstantBuffer(new(BufferLayout.Std140, setIndex, binding, name, type)); } - private void AddNewStorageBuffer(int binding, string name) + private void AddNewStorageBuffer(int setIndex, int binding, string name) { StructureType type = new(new[] { new StructureField(AggregateType.Array | AggregateType.U32, "data", 0), }); - Properties.AddOrUpdateStorageBuffer(new(BufferLayout.Std430, 1, binding, name, type)); + Properties.AddOrUpdateStorageBuffer(new(BufferLayout.Std430, setIndex, binding, name, type)); } public static string GetShaderStagePrefix(ShaderStage stage) diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceReservations.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceReservations.cs index d559f66990..c89c4d0b66 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceReservations.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/ResourceReservations.cs @@ -11,6 +11,8 @@ namespace Ryujinx.Graphics.Shader.Translation public const int MaxVertexBufferTextures = 32; + private const int TextureSetIndex = 2; // TODO: Get from GPU accessor. + public int VertexInfoConstantBufferBinding { get; } public int VertexOutputStorageBufferBinding { get; } public int GeometryVertexOutputStorageBufferBinding { get; } @@ -163,6 +165,21 @@ namespace Ryujinx.Graphics.Shader.Translation return _vertexBufferTextureBaseBinding + vaLocation; } + public SetBindingPair GetVertexBufferTextureSetAndBinding(int vaLocation) + { + return new SetBindingPair(TextureSetIndex, GetVertexBufferTextureBinding(vaLocation)); + } + + public SetBindingPair GetIndexBufferTextureSetAndBinding() + { + return new SetBindingPair(TextureSetIndex, IndexBufferTextureBinding); + } + + public SetBindingPair GetTopologyRemapBufferTextureSetAndBinding() + { + return new SetBindingPair(TextureSetIndex, TopologyRemapBufferTextureBinding); + } + internal bool TryGetOffset(StorageKind storageKind, int location, int component, out int offset) { return _offsets.TryGetValue(new IoDefinition(storageKind, IoVariable.UserDefined, location, component), out offset); diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Transforms/TexturePass.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Transforms/TexturePass.cs index 072b456955..6ba8cb44ad 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Transforms/TexturePass.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Transforms/TexturePass.cs @@ -182,6 +182,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms texOp.Type, texOp.Format, texOp.Flags, + texOp.Set, texOp.Binding, index, new[] { coordSize }, @@ -251,6 +252,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms texOp.Type, texOp.Format, texOp.Flags, + texOp.Set, texOp.Binding, index, new[] { coordSize }, @@ -471,6 +473,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms texOp.Type, texOp.Format, texOp.Flags & ~(TextureFlags.Offset | TextureFlags.Offsets), + texOp.Set, texOp.Binding, 1 << 3, // W component: i=0, j=0 new[] { dests[destIndex++] }, @@ -527,6 +530,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms texOp.Type, texOp.Format, texOp.Flags & ~(TextureFlags.Offset | TextureFlags.Offsets), + texOp.Set, texOp.Binding, componentIndex, dests, @@ -573,6 +577,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms texOp.Type, texOp.Format, texOp.Flags, + texOp.Set, texOp.Binding, index, new[] { texSizes[index] }, @@ -603,6 +608,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms texOp.Type, texOp.Format, texOp.Flags, + texOp.Set, texOp.Binding, 0, new[] { lod }, @@ -633,6 +639,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms texOp.Type, texOp.Format, texOp.Flags, + texOp.Set, texOp.Binding, index, new[] { texSizes[index] }, diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Transforms/VertexToCompute.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Transforms/VertexToCompute.cs index d71ada8658..ddd2134d2e 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Transforms/VertexToCompute.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/Transforms/VertexToCompute.cs @@ -54,6 +54,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms { bool needsSextNorm = context.Definitions.IsAttributePackedRgb10A2Signed(location); + SetBindingPair setAndBinding = context.ResourceManager.Reservations.GetVertexBufferTextureSetAndBinding(location); Operand temp = needsSextNorm ? Local() : dest; Operand vertexElemOffset = GenerateVertexOffset(context.ResourceManager, node, location, 0); @@ -62,7 +63,8 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms SamplerType.TextureBuffer, TextureFormat.Unknown, TextureFlags.IntCoords, - context.ResourceManager.Reservations.GetVertexBufferTextureBinding(location), + setAndBinding.SetIndex, + setAndBinding.Binding, 1 << component, new[] { temp }, new[] { vertexElemOffset })); @@ -75,6 +77,7 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms } else { + SetBindingPair setAndBinding = context.ResourceManager.Reservations.GetVertexBufferTextureSetAndBinding(location); Operand temp = component > 0 ? Local() : dest; Operand vertexElemOffset = GenerateVertexOffset(context.ResourceManager, node, location, component); @@ -83,7 +86,8 @@ namespace Ryujinx.Graphics.Shader.Translation.Transforms SamplerType.TextureBuffer, TextureFormat.Unknown, TextureFlags.IntCoords, - context.ResourceManager.Reservations.GetVertexBufferTextureBinding(location), + setAndBinding.SetIndex, + setAndBinding.Binding, 1, new[] { temp }, new[] { vertexElemOffset })); diff --git a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs index 1065355886..59914736ee 100644 --- a/ryujinx/src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs +++ b/ryujinx/src/Ryujinx.Graphics.Shader/Translation/TranslatorContext.cs @@ -412,8 +412,8 @@ namespace Ryujinx.Graphics.Shader.Translation if (Stage == ShaderStage.Vertex) { - int ibBinding = resourceManager.Reservations.IndexBufferTextureBinding; - TextureDefinition indexBuffer = new(2, ibBinding, "ib_data", SamplerType.TextureBuffer); + SetBindingPair ibSetAndBinding = resourceManager.Reservations.GetIndexBufferTextureSetAndBinding(); + TextureDefinition indexBuffer = new(ibSetAndBinding.SetIndex, ibSetAndBinding.Binding, "ib_data", SamplerType.TextureBuffer); resourceManager.Properties.AddOrUpdateTexture(indexBuffer); int inputMap = _program.AttributeUsage.UsedInputAttributes; @@ -421,8 +421,8 @@ namespace Ryujinx.Graphics.Shader.Translation while (inputMap != 0) { int location = BitOperations.TrailingZeroCount(inputMap); - int binding = resourceManager.Reservations.GetVertexBufferTextureBinding(location); - TextureDefinition vaBuffer = new(2, binding, $"vb_data{location}", SamplerType.TextureBuffer); + SetBindingPair setAndBinding = resourceManager.Reservations.GetVertexBufferTextureSetAndBinding(location); + TextureDefinition vaBuffer = new(setAndBinding.SetIndex, setAndBinding.Binding, $"vb_data{location}", SamplerType.TextureBuffer); resourceManager.Properties.AddOrUpdateTexture(vaBuffer); inputMap &= ~(1 << location); @@ -430,8 +430,8 @@ namespace Ryujinx.Graphics.Shader.Translation } else if (Stage == ShaderStage.Geometry) { - int trbBinding = resourceManager.Reservations.TopologyRemapBufferTextureBinding; - TextureDefinition remapBuffer = new(2, trbBinding, "trb_data", SamplerType.TextureBuffer); + SetBindingPair trbSetAndBinding = resourceManager.Reservations.GetTopologyRemapBufferTextureSetAndBinding(); + TextureDefinition remapBuffer = new(trbSetAndBinding.SetIndex, trbSetAndBinding.Binding, "trb_data", SamplerType.TextureBuffer); resourceManager.Properties.AddOrUpdateTexture(remapBuffer); int geometryVbOutputSbBinding = resourceManager.Reservations.GeometryVertexOutputStorageBufferBinding; diff --git a/ryujinx/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs b/ryujinx/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs index a0010e660e..382f88d053 100644 --- a/ryujinx/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs +++ b/ryujinx/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs @@ -69,17 +69,7 @@ namespace Ryujinx.Graphics.Vulkan } } - private record struct ArrayRef - { - public ShaderStage Stage; - public T Array; - - public ArrayRef(ShaderStage stage, T array) - { - Stage = stage; - Array = array; - } - } + private readonly record struct ArrayRef(ShaderStage Stage, T Array); private readonly VulkanRenderer _gd; private readonly Device _device; @@ -97,6 +87,9 @@ namespace Ryujinx.Graphics.Vulkan private ArrayRef[] _textureArrayRefs; private ArrayRef[] _imageArrayRefs; + private ArrayRef[] _textureArrayExtraRefs; + private ArrayRef[] _imageArrayExtraRefs; + private readonly DescriptorBufferInfo[] _uniformBuffers; private readonly DescriptorBufferInfo[] _storageBuffers; private readonly DescriptorImageInfo[] _textures; @@ -152,6 +145,9 @@ namespace Ryujinx.Graphics.Vulkan _textureArrayRefs = Array.Empty>(); _imageArrayRefs = Array.Empty>(); + _textureArrayExtraRefs = Array.Empty>(); + _imageArrayExtraRefs = Array.Empty>(); + _uniformBuffers = new DescriptorBufferInfo[Constants.MaxUniformBufferBindings]; _storageBuffers = new DescriptorBufferInfo[Constants.MaxStorageBufferBindings]; _textures = new DescriptorImageInfo[Constants.MaxTexturesPerStage]; @@ -495,25 +491,39 @@ namespace Ryujinx.Graphics.Vulkan public void SetTextureArray(CommandBufferScoped cbs, ShaderStage stage, int binding, ITextureArray array) { - if (_textureArrayRefs.Length <= binding) - { - Array.Resize(ref _textureArrayRefs, binding + ArrayGrowthSize); - } + ref ArrayRef arrayRef = ref GetArrayRef(ref _textureArrayRefs, binding, ArrayGrowthSize); - if (_textureArrayRefs[binding].Stage != stage || _textureArrayRefs[binding].Array != array) + if (arrayRef.Stage != stage || arrayRef.Array != array) { - if (_textureArrayRefs[binding].Array != null) - { - _textureArrayRefs[binding].Array.Bound = false; - } + arrayRef.Array?.DecrementBindCount(); if (array is TextureArray textureArray) { - textureArray.Bound = true; + textureArray.IncrementBindCount(); textureArray.QueueWriteToReadBarriers(cbs, stage.ConvertToPipelineStageFlags()); } - _textureArrayRefs[binding] = new ArrayRef(stage, array as TextureArray); + arrayRef = new ArrayRef(stage, array as TextureArray); + + SignalDirty(DirtyFlags.Texture); + } + } + + public void SetTextureArraySeparate(CommandBufferScoped cbs, ShaderStage stage, int setIndex, ITextureArray array) + { + ref ArrayRef arrayRef = ref GetArrayRef(ref _textureArrayExtraRefs, setIndex - PipelineBase.DescriptorSetLayouts); + + if (arrayRef.Stage != stage || arrayRef.Array != array) + { + arrayRef.Array?.DecrementBindCount(); + + if (array is TextureArray textureArray) + { + textureArray.IncrementBindCount(); + textureArray.QueueWriteToReadBarriers(cbs, stage.ConvertToPipelineStageFlags()); + } + + arrayRef = new ArrayRef(stage, array as TextureArray); SignalDirty(DirtyFlags.Texture); } @@ -521,30 +531,56 @@ namespace Ryujinx.Graphics.Vulkan public void SetImageArray(CommandBufferScoped cbs, ShaderStage stage, int binding, IImageArray array) { - if (_imageArrayRefs.Length <= binding) - { - Array.Resize(ref _imageArrayRefs, binding + ArrayGrowthSize); - } + ref ArrayRef arrayRef = ref GetArrayRef(ref _imageArrayRefs, binding, ArrayGrowthSize); - if (_imageArrayRefs[binding].Stage != stage || _imageArrayRefs[binding].Array != array) + if (arrayRef.Stage != stage || arrayRef.Array != array) { - if (_imageArrayRefs[binding].Array != null) - { - _imageArrayRefs[binding].Array.Bound = false; - } + arrayRef.Array?.DecrementBindCount(); if (array is ImageArray imageArray) { - imageArray.Bound = true; + imageArray.IncrementBindCount(); imageArray.QueueWriteToReadBarriers(cbs, stage.ConvertToPipelineStageFlags()); } - _imageArrayRefs[binding] = new ArrayRef(stage, array as ImageArray); + arrayRef = new ArrayRef(stage, array as ImageArray); SignalDirty(DirtyFlags.Image); } } + public void SetImageArraySeparate(CommandBufferScoped cbs, ShaderStage stage, int setIndex, IImageArray array) + { + ref ArrayRef arrayRef = ref GetArrayRef(ref _imageArrayExtraRefs, setIndex - PipelineBase.DescriptorSetLayouts); + + if (arrayRef.Stage != stage || arrayRef.Array != array) + { + arrayRef.Array?.DecrementBindCount(); + + if (array is ImageArray imageArray) + { + imageArray.IncrementBindCount(); + imageArray.QueueWriteToReadBarriers(cbs, stage.ConvertToPipelineStageFlags()); + } + + arrayRef = new ArrayRef(stage, array as ImageArray); + + SignalDirty(DirtyFlags.Image); + } + } + + private static ref ArrayRef GetArrayRef(ref ArrayRef[] array, int index, int growthSize = 1) + { + ArgumentOutOfRangeException.ThrowIfNegative(index); + + if (array.Length <= index) + { + Array.Resize(ref array, index + growthSize); + } + + return ref array[index]; + } + public void SetUniformBuffers(CommandBuffer commandBuffer, ReadOnlySpan buffers) { for (int i = 0; i < buffers.Length; i++) @@ -594,31 +630,40 @@ namespace Ryujinx.Graphics.Vulkan return; } + var program = _program; + if (_dirty.HasFlag(DirtyFlags.Uniform)) { - if (_program.UsePushDescriptors) + if (program.UsePushDescriptors) { - UpdateAndBindUniformBufferPd(cbs, pbp); + UpdateAndBindUniformBufferPd(cbs); } else { - UpdateAndBind(cbs, PipelineBase.UniformSetIndex, pbp); + UpdateAndBind(cbs, program, PipelineBase.UniformSetIndex, pbp); } } if (_dirty.HasFlag(DirtyFlags.Storage)) { - UpdateAndBind(cbs, PipelineBase.StorageSetIndex, pbp); + UpdateAndBind(cbs, program, PipelineBase.StorageSetIndex, pbp); } if (_dirty.HasFlag(DirtyFlags.Texture)) { - UpdateAndBind(cbs, PipelineBase.TextureSetIndex, pbp); + UpdateAndBind(cbs, program, PipelineBase.TextureSetIndex, pbp); } if (_dirty.HasFlag(DirtyFlags.Image)) { - UpdateAndBind(cbs, PipelineBase.ImageSetIndex, pbp); + UpdateAndBind(cbs, program, PipelineBase.ImageSetIndex, pbp); + } + + if (program.BindingSegments.Length > PipelineBase.DescriptorSetLayouts) + { + // Program is using extra sets, we need to bind those too. + + BindExtraSets(cbs, program, pbp); } _dirty = DirtyFlags.None; @@ -658,9 +703,8 @@ namespace Ryujinx.Graphics.Vulkan } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void UpdateAndBind(CommandBufferScoped cbs, int setIndex, PipelineBindPoint pbp) + private void UpdateAndBind(CommandBufferScoped cbs, ShaderCollection program, int setIndex, PipelineBindPoint pbp) { - var program = _program; var bindingSegments = program.BindingSegments[setIndex]; if (bindingSegments.Length == 0) @@ -869,7 +913,7 @@ namespace Ryujinx.Graphics.Vulkan } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void UpdateAndBindUniformBufferPd(CommandBufferScoped cbs, PipelineBindPoint pbp) + private void UpdateAndBindUniformBufferPd(CommandBufferScoped cbs) { int sequence = _pdSequence; var bindingSegments = _program.BindingSegments[PipelineBase.UniformSetIndex]; @@ -933,6 +977,56 @@ namespace Ryujinx.Graphics.Vulkan } } + private void BindExtraSets(CommandBufferScoped cbs, ShaderCollection program, PipelineBindPoint pbp) + { + for (int setIndex = PipelineBase.DescriptorSetLayouts; setIndex < program.BindingSegments.Length; setIndex++) + { + var bindingSegments = program.BindingSegments[setIndex]; + + if (bindingSegments.Length == 0) + { + continue; + } + + ResourceBindingSegment segment = bindingSegments[0]; + + if (segment.IsArray) + { + DescriptorSet[] sets = null; + + if (segment.Type == ResourceType.Texture || + segment.Type == ResourceType.Sampler || + segment.Type == ResourceType.TextureAndSampler || + segment.Type == ResourceType.BufferTexture) + { + sets = _textureArrayExtraRefs[setIndex - PipelineBase.DescriptorSetLayouts].Array.GetDescriptorSets( + _device, + cbs, + _templateUpdater, + program, + setIndex, + _dummyTexture, + _dummySampler); + } + else if (segment.Type == ResourceType.Image || segment.Type == ResourceType.BufferImage) + { + sets = _imageArrayExtraRefs[setIndex - PipelineBase.DescriptorSetLayouts].Array.GetDescriptorSets( + _device, + cbs, + _templateUpdater, + program, + setIndex, + _dummyTexture); + } + + if (sets != null) + { + _gd.Api.CmdBindDescriptorSets(cbs.CommandBuffer, pbp, _program.PipelineLayout, (uint)setIndex, 1, sets, 0, ReadOnlySpan.Empty); + } + } + } + } + public void SignalCommandBufferChange() { _updateDescriptorCacheCbIndex = true; diff --git a/ryujinx/src/Ryujinx.Graphics.Vulkan/ImageArray.cs b/ryujinx/src/Ryujinx.Graphics.Vulkan/ImageArray.cs index 38a5b6b481..3c7f321ff7 100644 --- a/ryujinx/src/Ryujinx.Graphics.Vulkan/ImageArray.cs +++ b/ryujinx/src/Ryujinx.Graphics.Vulkan/ImageArray.cs @@ -2,6 +2,7 @@ using Ryujinx.Graphics.GAL; using Silk.NET.Vulkan; using System; using System.Collections.Generic; +using System.Diagnostics; namespace Ryujinx.Graphics.Vulkan { @@ -24,12 +25,18 @@ namespace Ryujinx.Graphics.Vulkan private HashSet _storages; + private DescriptorSet[] _cachedDescriptorSets; + private int _cachedCommandBufferIndex; private int _cachedSubmissionCount; + private ShaderCollection _cachedDscProgram; + private int _cachedDscSetIndex; + private int _cachedDscIndex; + private readonly bool _isBuffer; - public bool Bound; + private int _bindCount; public ImageArray(VulkanRenderer gd, int size, bool isBuffer) { @@ -97,8 +104,12 @@ namespace Ryujinx.Graphics.Vulkan { _cachedCommandBufferIndex = -1; _storages = null; + _cachedDescriptorSets = null; - _gd.PipelineInternal.ForceImageDirty(); + if (_bindCount != 0) + { + _gd.PipelineInternal.ForceImageDirty(); + } } public void QueueWriteToReadBarriers(CommandBufferScoped cbs, PipelineStageFlags stageFlags) @@ -175,5 +186,65 @@ namespace Ryujinx.Graphics.Vulkan return bufferTextures; } + + public DescriptorSet[] GetDescriptorSets( + Device device, + CommandBufferScoped cbs, + DescriptorSetTemplateUpdater templateUpdater, + ShaderCollection program, + int setIndex, + TextureView dummyTexture) + { + if (_cachedDescriptorSets != null) + { + // We still need to ensure the current command buffer holds a reference to all used textures. + + if (!_isBuffer) + { + GetImageInfos(_gd, cbs, dummyTexture); + } + else + { + GetBufferViews(cbs); + } + + return _cachedDescriptorSets; + } + + _cachedDscProgram?.ReleaseManualDescriptorSetCollection(_cachedDscSetIndex, _cachedDscIndex); + var dsc = program.GetNewManualDescriptorSetCollection(cbs.CommandBufferIndex, setIndex, out _cachedDscIndex).Get(cbs); + + DescriptorSetTemplate template = program.Templates[setIndex]; + + DescriptorSetTemplateWriter tu = templateUpdater.Begin(template); + + if (!_isBuffer) + { + tu.Push(GetImageInfos(_gd, cbs, dummyTexture)); + } + else + { + tu.Push(GetBufferViews(cbs)); + } + + var sets = dsc.GetSets(); + templateUpdater.Commit(_gd, device, sets[0]); + _cachedDescriptorSets = sets; + _cachedDscProgram = program; + _cachedDscSetIndex = setIndex; + + return sets; + } + + public void IncrementBindCount() + { + _bindCount++; + } + + public void DecrementBindCount() + { + int newBindCount = --_bindCount; + Debug.Assert(newBindCount >= 0); + } } } diff --git a/ryujinx/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs b/ryujinx/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs index 3776e2f690..918de59b73 100644 --- a/ryujinx/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs +++ b/ryujinx/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs @@ -751,14 +751,12 @@ namespace Ryujinx.Graphics.Vulkan _vertexBufferUpdater.Commit(Cbs); } -#pragma warning disable CA1822 // Mark member as static public void SetAlphaTest(bool enable, float reference, CompareOp op) { // This is currently handled using shader specialization, as Vulkan does not support alpha test. // In the future, we may want to use this to write the reference value into the support buffer, // to avoid creating one version of the shader per reference value used. } -#pragma warning restore CA1822 public void SetBlendState(AdvancedBlendDescriptor blend) { @@ -903,6 +901,11 @@ namespace Ryujinx.Graphics.Vulkan _descriptorSetUpdater.SetImageArray(Cbs, stage, binding, array); } + public void SetImageArraySeparate(ShaderStage stage, int setIndex, IImageArray array) + { + _descriptorSetUpdater.SetImageArraySeparate(Cbs, stage, setIndex, array); + } + public void SetIndexBuffer(BufferRange buffer, IndexType type) { if (buffer.Handle != BufferHandle.Null) @@ -945,7 +948,6 @@ namespace Ryujinx.Graphics.Vulkan // TODO: Default levels (likely needs emulation on shaders?) } -#pragma warning disable CA1822 // Mark member as static public void SetPointParameters(float size, bool isProgramPointSize, bool enablePointSprite, Origin origin) { // TODO. @@ -955,7 +957,6 @@ namespace Ryujinx.Graphics.Vulkan { // TODO. } -#pragma warning restore CA1822 public void SetPrimitiveRestart(bool enable, int index) { @@ -1156,6 +1157,11 @@ namespace Ryujinx.Graphics.Vulkan _descriptorSetUpdater.SetTextureArray(Cbs, stage, binding, array); } + public void SetTextureArraySeparate(ShaderStage stage, int setIndex, ITextureArray array) + { + _descriptorSetUpdater.SetTextureArraySeparate(Cbs, stage, setIndex, array); + } + public void SetTransformFeedbackBuffers(ReadOnlySpan buffers) { PauseTransformFeedbackInternal(); @@ -1186,12 +1192,10 @@ namespace Ryujinx.Graphics.Vulkan _descriptorSetUpdater.SetUniformBuffers(CommandBuffer, buffers); } -#pragma warning disable CA1822 // Mark member as static public void SetUserClipDistance(int index, bool enableClip) { // TODO. } -#pragma warning restore CA1822 public void SetVertexAttribs(ReadOnlySpan vertexAttribs) { diff --git a/ryujinx/src/Ryujinx.Graphics.Vulkan/PipelineLayoutCacheEntry.cs b/ryujinx/src/Ryujinx.Graphics.Vulkan/PipelineLayoutCacheEntry.cs index fb1f0a5ff6..7d0948d6e6 100644 --- a/ryujinx/src/Ryujinx.Graphics.Vulkan/PipelineLayoutCacheEntry.cs +++ b/ryujinx/src/Ryujinx.Graphics.Vulkan/PipelineLayoutCacheEntry.cs @@ -3,6 +3,7 @@ using Silk.NET.Vulkan; using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Runtime.InteropServices; namespace Ryujinx.Graphics.Vulkan { @@ -27,6 +28,24 @@ namespace Ryujinx.Graphics.Vulkan private int _dsLastCbIndex; private int _dsLastSubmissionCount; + private struct ManualDescriptorSetEntry + { + public Auto DescriptorSet; + public int CbIndex; + public int CbSubmissionCount; + public bool InUse; + + public ManualDescriptorSetEntry(Auto descriptorSet, int cbIndex, int cbSubmissionCount, bool inUse) + { + DescriptorSet = descriptorSet; + CbIndex = cbIndex; + CbSubmissionCount = cbSubmissionCount; + InUse = inUse; + } + } + + private readonly List[] _manualDsCache; + private readonly Dictionary _pdTemplates; private readonly ResourceDescriptorCollection _pdDescriptors; private long _lastPdUsage; @@ -50,6 +69,7 @@ namespace Ryujinx.Graphics.Vulkan } _dsCacheCursor = new int[setsCount]; + _manualDsCache = new List[setsCount]; } public PipelineLayoutCacheEntry( @@ -124,6 +144,51 @@ namespace Ryujinx.Graphics.Vulkan return list[index]; } + public Auto GetNewManualDescriptorSetCollection(int commandBufferIndex, int setIndex, out int cacheIndex) + { + int submissionCount = _gd.CommandBufferPool.GetSubmissionCount(commandBufferIndex); + + var list = _manualDsCache[setIndex] ??= new(); + var span = CollectionsMarshal.AsSpan(list); + + for (int index = 0; index < span.Length; index++) + { + ref ManualDescriptorSetEntry entry = ref span[index]; + + if (!entry.InUse && (entry.CbIndex != commandBufferIndex || entry.CbSubmissionCount != submissionCount)) + { + entry.InUse = true; + entry.CbIndex = commandBufferIndex; + entry.CbSubmissionCount = submissionCount; + + cacheIndex = index; + + return entry.DescriptorSet; + } + } + + var dsc = _descriptorSetManager.AllocateDescriptorSet( + _gd.Api, + DescriptorSetLayouts[setIndex], + _poolSizes[setIndex], + setIndex, + _consumedDescriptorsPerSet[setIndex], + false); + + cacheIndex = list.Count; + list.Add(new ManualDescriptorSetEntry(dsc, commandBufferIndex, submissionCount, inUse: true)); + + return dsc; + } + + public void ReleaseManualDescriptorSetCollection(int setIndex, int cacheIndex) + { + var list = _manualDsCache[setIndex]; + var span = CollectionsMarshal.AsSpan(list); + + span[cacheIndex].InUse = false; + } + private static Span GetDescriptorPoolSizes(Span output, ResourceDescriptorCollection setDescriptor, uint multiplier) { int count = 0; @@ -204,6 +269,21 @@ namespace Ryujinx.Graphics.Vulkan } } + for (int i = 0; i < _manualDsCache.Length; i++) + { + if (_manualDsCache[i] == null) + { + continue; + } + + for (int j = 0; j < _manualDsCache[i].Count; j++) + { + _manualDsCache[i][j].DescriptorSet.Dispose(); + } + + _manualDsCache[i].Clear(); + } + _gd.Api.DestroyPipelineLayout(_device, PipelineLayout, null); for (int i = 0; i < DescriptorSetLayouts.Length; i++) diff --git a/ryujinx/src/Ryujinx.Graphics.Vulkan/ShaderCollection.cs b/ryujinx/src/Ryujinx.Graphics.Vulkan/ShaderCollection.cs index 1785469839..f2d648a517 100644 --- a/ryujinx/src/Ryujinx.Graphics.Vulkan/ShaderCollection.cs +++ b/ryujinx/src/Ryujinx.Graphics.Vulkan/ShaderCollection.cs @@ -604,6 +604,16 @@ namespace Ryujinx.Graphics.Vulkan return _plce.GetNewDescriptorSetCollection(setIndex, out isNew); } + public Auto GetNewManualDescriptorSetCollection(int commandBufferIndex, int setIndex, out int cacheIndex) + { + return _plce.GetNewManualDescriptorSetCollection(commandBufferIndex, setIndex, out cacheIndex); + } + + public void ReleaseManualDescriptorSetCollection(int setIndex, int cacheIndex) + { + _plce.ReleaseManualDescriptorSetCollection(setIndex, cacheIndex); + } + public bool HasSameLayout(ShaderCollection other) { return other != null && _plce == other._plce; diff --git a/ryujinx/src/Ryujinx.Graphics.Vulkan/TextureArray.cs b/ryujinx/src/Ryujinx.Graphics.Vulkan/TextureArray.cs index 6ef9087bcd..fe834225c5 100644 --- a/ryujinx/src/Ryujinx.Graphics.Vulkan/TextureArray.cs +++ b/ryujinx/src/Ryujinx.Graphics.Vulkan/TextureArray.cs @@ -2,6 +2,7 @@ using Ryujinx.Graphics.GAL; using Silk.NET.Vulkan; using System; using System.Collections.Generic; +using System.Diagnostics; namespace Ryujinx.Graphics.Vulkan { @@ -24,12 +25,18 @@ namespace Ryujinx.Graphics.Vulkan private HashSet _storages; + private DescriptorSet[] _cachedDescriptorSets; + private int _cachedCommandBufferIndex; private int _cachedSubmissionCount; + private ShaderCollection _cachedDscProgram; + private int _cachedDscSetIndex; + private int _cachedDscIndex; + private readonly bool _isBuffer; - public bool Bound; + private int _bindCount; public TextureArray(VulkanRenderer gd, int size, bool isBuffer) { @@ -106,8 +113,12 @@ namespace Ryujinx.Graphics.Vulkan { _cachedCommandBufferIndex = -1; _storages = null; + _cachedDescriptorSets = null; - _gd.PipelineInternal.ForceTextureDirty(); + if (_bindCount != 0) + { + _gd.PipelineInternal.ForceTextureDirty(); + } } public void QueueWriteToReadBarriers(CommandBufferScoped cbs, PipelineStageFlags stageFlags) @@ -190,5 +201,66 @@ namespace Ryujinx.Graphics.Vulkan return bufferTextures; } + + public DescriptorSet[] GetDescriptorSets( + Device device, + CommandBufferScoped cbs, + DescriptorSetTemplateUpdater templateUpdater, + ShaderCollection program, + int setIndex, + TextureView dummyTexture, + SamplerHolder dummySampler) + { + if (_cachedDescriptorSets != null) + { + // We still need to ensure the current command buffer holds a reference to all used textures. + + if (!_isBuffer) + { + GetImageInfos(_gd, cbs, dummyTexture, dummySampler); + } + else + { + GetBufferViews(cbs); + } + + return _cachedDescriptorSets; + } + + _cachedDscProgram?.ReleaseManualDescriptorSetCollection(_cachedDscSetIndex, _cachedDscIndex); + var dsc = program.GetNewManualDescriptorSetCollection(cbs.CommandBufferIndex, setIndex, out _cachedDscIndex).Get(cbs); + + DescriptorSetTemplate template = program.Templates[setIndex]; + + DescriptorSetTemplateWriter tu = templateUpdater.Begin(template); + + if (!_isBuffer) + { + tu.Push(GetImageInfos(_gd, cbs, dummyTexture, dummySampler)); + } + else + { + tu.Push(GetBufferViews(cbs)); + } + + var sets = dsc.GetSets(); + templateUpdater.Commit(_gd, device, sets[0]); + _cachedDescriptorSets = sets; + _cachedDscProgram = program; + _cachedDscSetIndex = setIndex; + + return sets; + } + + public void IncrementBindCount() + { + _bindCount++; + } + + public void DecrementBindCount() + { + int newBindCount = --_bindCount; + Debug.Assert(newBindCount >= 0); + } } } diff --git a/ryujinx/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs b/ryujinx/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs index 175d5e3ea3..86a347e019 100644 --- a/ryujinx/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs +++ b/ryujinx/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs @@ -728,6 +728,12 @@ namespace Ryujinx.Graphics.Vulkan supportsViewportSwizzle: false, supportsIndirectParameters: true, supportsDepthClipControl: Capabilities.SupportsDepthClipControl, + uniformBufferSetIndex: PipelineBase.UniformSetIndex, + storageBufferSetIndex: PipelineBase.StorageSetIndex, + textureSetIndex: PipelineBase.TextureSetIndex, + imageSetIndex: PipelineBase.ImageSetIndex, + extraSetBaseIndex: PipelineBase.DescriptorSetLayouts, + maximumExtraSets: Math.Max(0, (int)limits.MaxBoundDescriptorSets - PipelineBase.DescriptorSetLayouts), maximumUniformBuffersPerStage: Constants.MaxUniformBuffersPerStage, maximumStorageBuffersPerStage: Constants.MaxStorageBuffersPerStage, maximumTexturesPerStage: Constants.MaxTexturesPerStage, diff --git a/ryujinx/src/Ryujinx.ShaderTools/Program.cs b/ryujinx/src/Ryujinx.ShaderTools/Program.cs index d2c6bd59e4..a84d7b4664 100644 --- a/ryujinx/src/Ryujinx.ShaderTools/Program.cs +++ b/ryujinx/src/Ryujinx.ShaderTools/Program.cs @@ -25,32 +25,32 @@ namespace Ryujinx.ShaderTools _imagesCount = 0; } - public int CreateConstantBufferBinding(int index) + public SetBindingPair CreateConstantBufferBinding(int index) { - return index + 1; + return new SetBindingPair(0, index + 1); } - public int CreateImageBinding(int count, bool isBuffer) + public SetBindingPair CreateImageBinding(int count, bool isBuffer) { int binding = _imagesCount; _imagesCount += count; - return binding; + return new SetBindingPair(3, binding); } - public int CreateStorageBufferBinding(int index) + public SetBindingPair CreateStorageBufferBinding(int index) { - return index; + return new SetBindingPair(1, index); } - public int CreateTextureBinding(int count, bool isBuffer) + public SetBindingPair CreateTextureBinding(int count, bool isBuffer) { int binding = _texturesCount; _texturesCount += count; - return binding; + return new SetBindingPair(2, binding); } public ReadOnlySpan GetCode(ulong address, int minimumSize) diff --git a/shadowsocks-rust/.github/workflows/build-release.yml b/shadowsocks-rust/.github/workflows/build-release.yml index 17b97643c0..f9f5955ba8 100644 --- a/shadowsocks-rust/.github/workflows/build-release.yml +++ b/shadowsocks-rust/.github/workflows/build-release.yml @@ -23,13 +23,13 @@ jobs: matrix: target: - i686-unknown-linux-musl - - x86_64-pc-windows-gnu + # - x86_64-pc-windows-gnu - x86_64-unknown-linux-gnu - x86_64-unknown-linux-musl - - armv7-unknown-linux-musleabihf - - armv7-unknown-linux-gnueabihf + # - armv7-unknown-linux-musleabihf + # - armv7-unknown-linux-gnueabihf - arm-unknown-linux-gnueabi - - arm-unknown-linux-gnueabihf + # - arm-unknown-linux-gnueabihf - arm-unknown-linux-musleabi - arm-unknown-linux-musleabihf - aarch64-unknown-linux-gnu diff --git a/shadowsocks-rust/Dockerfile b/shadowsocks-rust/Dockerfile index 61b3513291..06df73794e 100644 --- a/shadowsocks-rust/Dockerfile +++ b/shadowsocks-rust/Dockerfile @@ -1,9 +1,9 @@ -FROM --platform=$BUILDPLATFORM rust:1.67.1-alpine3.17 AS builder +FROM --platform=$BUILDPLATFORM rust:alpine3.20 AS builder ARG TARGETARCH RUN set -x \ - && apk add --no-cache build-base cmake + && apk add --no-cache build-base cmake llvm15-dev clang15-libclang clang15 rust-bindgen WORKDIR /root/shadowsocks-rust @@ -33,11 +33,10 @@ RUN case "$TARGETARCH" in \ && echo "CC=$CC" \ && rustup override set stable \ && rustup target add "$RUST_TARGET" \ - && cargo install --force --locked bindgen-cli \ && RUSTFLAGS="-C linker=$CC" CC=$CC cargo build --target "$RUST_TARGET" --release --features "full" \ && mv target/$RUST_TARGET/release/ss* target/release/ -FROM alpine:3.17 AS sslocal +FROM alpine:3.20 AS sslocal # NOTE: Please be careful to change the path of these binaries, refer to #1149 for more information. COPY --from=builder /root/shadowsocks-rust/target/release/sslocal /usr/bin/ diff --git a/shadowsocks-rust/README.md b/shadowsocks-rust/README.md index f7b7ec23f8..60179e4c08 100644 --- a/shadowsocks-rust/README.md +++ b/shadowsocks-rust/README.md @@ -649,6 +649,25 @@ Example configuration: // Linux/Android: tproxy (default) // FreeBSD/OpenBSD: pf (default) "udp_redir": "tproxy" + }, + { + // FakeDNS local server (feature = "local-fake-dns") + // FakeDNS is a DNS server that allocates an IPv4 / IPv6 address in a specific pool for each queries. + // Subsequence requests from the other local interfaces that the target addresses includes those allocated IP addresses, + // will be substituted back to their original domain name addresses. + // This feature is useful mostly for transparent proxy, which will allow the proxied domain names to be resolved remotely. + "protocol": "fake-dns", + // Listen address + "local_address": "127.0.0.1", + "local_port": 10053, + // IPv4 address pool (for A records) + "fake_dns_ipv4_network": "10.255.0.0/16", + // IPv6 address pool (for AAAA records) + "fake_dns_ipv6_network": "fdf2:e786:ab40:9d2f::/64", + // Persistent storage for all allocated DNS records + "fake_dns_database_path": "/var/shadowsocks/fakedns.db", + // OPTIONAL: Record expire duration in seconds, 10s by default + "fake_dns_record_expire_duration": 10 } ], diff --git a/shadowsocks-rust/snap/snapcraft.yaml b/shadowsocks-rust/snap/snapcraft.yaml index 63069a20fb..e241a78fec 100644 --- a/shadowsocks-rust/snap/snapcraft.yaml +++ b/shadowsocks-rust/snap/snapcraft.yaml @@ -62,6 +62,9 @@ parts: override-pull: | snapcraftctl pull snapcraftctl set-version `git describe --tags --long | sed 's/\([^-]*-g\)/r\1/;s/-/./g'` - -build-packages: - - protobuf + build-packages: + - cmake + - bindgen + - llvm-dev + - libclang-dev + - clang diff --git a/small/dns2tcp/Makefile b/small/dns2tcp/Makefile index 7d161ee0f3..5ed3b15765 100644 --- a/small/dns2tcp/Makefile +++ b/small/dns2tcp/Makefile @@ -5,21 +5,19 @@ include $(TOPDIR)/rules.mk PKG_NAME:=dns2tcp -PKG_VERSION:=1.1.0 +PKG_VERSION:=1.1.1 PKG_RELEASE:=2 -PKG_SOURCE_PROTO:=git -PKG_SOURCE_URL:=https://github.com/zfl9/dns2tcp.git -PKG_SOURCE_DATE:=2023-04-15 -PKG_SOURCE_VERSION:=558b07e4c1f9bd5be19e60892a0bce7661ed99a5 -PKG_MIRROR_HASH:=872b420cb0d555390ce03a6bf6df32d33b33cdc9a734fac1f758d6d65e8adf7f +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/zfl9/dns2tcp/tar.gz/v$(PKG_VERSION)? +PKG_HASH:=35251fbe1645601086f21cdbd5a2f75471d812f99ed8017bb05158840456b43c PKG_MAINTAINER:=Tianling Shen PKG_LICENSE:=AGPL-3.0-only PKG_LICENSE_FILES:=LICENSE PKG_BUILD_PARALLEL:=1 -PKG_USE_MIPS16:=0 +PKG_BUILD_FLAGS:=no-mips16 gc-sections lto include $(INCLUDE_DIR)/package.mk @@ -31,7 +29,7 @@ define Package/dns2tcp URL:=https://github.com/zfl9/dns2tcp endef -TARGET_CFLAGS+= $(FPIC) -flto +TARGET_CFLAGS+= $(FPIC) MAKE_FLAGS+= \ CFLAGS="-std=c99 $(TARGET_CFLAGS)" \ EVCFLAGS="$(TARGET_CFLAGS)" diff --git a/small/shadowsocks-rust/Makefile b/small/shadowsocks-rust/Makefile index 0c1213bab8..b0ebc98dda 100644 --- a/small/shadowsocks-rust/Makefile +++ b/small/shadowsocks-rust/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=shadowsocks-rust -PKG_VERSION:=1.18.4 +PKG_VERSION:=1.19.0 PKG_RELEASE:=1 PKG_SOURCE_HEADER:=shadowsocks-v$(PKG_VERSION) @@ -21,29 +21,29 @@ endif ifeq ($(ARCH),aarch64) PKG_SOURCE:=$(PKG_SOURCE_HEADER).aarch64-$(PKG_SOURCE_BODY).$(PKG_SOURCE_FOOTER) - PKG_HASH:=7e30518c61917937e097e5525166aeb8b72f2bd8ab95382dba8b0374a7c77c06 + PKG_HASH:=c0ac84f0efdae92b19e72f5477438da5d2c835d3e0bf14a2b8dea0e0b6bcc064 else ifeq ($(ARCH),arm) # Referred to golang/golang-values.mk ARM_CPU_FEATURES:=$(word 2,$(subst +,$(space),$(call qstrip,$(CONFIG_CPU_TYPE)))) ifeq ($(ARM_CPU_FEATURES),) PKG_SOURCE:=$(PKG_SOURCE_HEADER).arm-$(PKG_SOURCE_BODY)eabi.$(PKG_SOURCE_FOOTER) - PKG_HASH:=257a148b921649c7ec24ce1d2ef0418c2c4554884b7fa3f8f6215d13ec2a56f7 + PKG_HASH:=2d350f6d990cfd0f0af70ec889e77bf121f7a711129bb2ff7def926676cb90ee else PKG_SOURCE:=$(PKG_SOURCE_HEADER).arm-$(PKG_SOURCE_BODY)eabihf.$(PKG_SOURCE_FOOTER) - PKG_HASH:=8e5e8a7bb342cb427ea649384a3024fe88fff8351c4880ef1b34cab2e4550440 + PKG_HASH:=da2ace727b75039aabc4ebff74bc1ade07ffd21e46d4239aa7537b8665f71d33 endif else ifeq ($(ARCH),i386) PKG_SOURCE:=$(PKG_SOURCE_HEADER).i686-$(PKG_SOURCE_BODY).$(PKG_SOURCE_FOOTER) - PKG_HASH:=194ea6282a54b846e8fb4675cc016741fb7433a02a5d6dd4f6184862dec1f737 + PKG_HASH:=074ac1591be87ba846df962f895b0dd7b07120ef900a1fbfee86b40f8f13d93d else ifeq ($(ARCH),x86_64) PKG_SOURCE:=$(PKG_SOURCE_HEADER).x86_64-$(PKG_SOURCE_BODY).$(PKG_SOURCE_FOOTER) - PKG_HASH:=f6aa7f2ce3b643344ccb39dcd419c4b4d39f19a9964ffa9b9bcffc3ec55fce59 + PKG_HASH:=b2321bd795db6fb71b3bc70da03e1dd6aa64d194ce3183ffda789e08329edfd1 else ifeq ($(ARCH),mips) PKG_SOURCE:=$(PKG_SOURCE_HEADER).mips-$(PKG_SOURCE_BODY).$(PKG_SOURCE_FOOTER) - PKG_HASH:=6416510d5ca1622090888770c76216cb0b637e4e57afe89aea1ac5a01945451d + PKG_HASH:=e5ba11eb81c94d5376586df8ad05793cf87038c83c2b02a6a1fbd3aebd6aecf8 else ifeq ($(ARCH),mipsel) PKG_SOURCE:=$(PKG_SOURCE_HEADER).mipsel-$(PKG_SOURCE_BODY).$(PKG_SOURCE_FOOTER) - PKG_HASH:=05db5387245582788f7538fa0c494217d197875bf9c00306ee0208d602bfe8b2 + PKG_HASH:=e735d637f3c2e57b1d102f1a0c64bc3688f59632b7aa946310d5338117e308fd # Set the default value to make OpenWrt Package Checker happy else PKG_SOURCE:=dummy diff --git a/xray-core/app/commander/config.pb.go b/xray-core/app/commander/config.pb.go index ecc279e92a..7601b74c8e 100644 --- a/xray-core/app/commander/config.pb.go +++ b/xray-core/app/commander/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 -// protoc v4.25.3 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/commander/config.proto package commander diff --git a/xray-core/app/dispatcher/config.pb.go b/xray-core/app/dispatcher/config.pb.go index 92936a7f6f..13af96ea7e 100644 --- a/xray-core/app/dispatcher/config.pb.go +++ b/xray-core/app/dispatcher/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/dispatcher/config.proto package dispatcher diff --git a/xray-core/app/dns/config.pb.go b/xray-core/app/dns/config.pb.go index 1c07244306..c37190c806 100644 --- a/xray-core/app/dns/config.pb.go +++ b/xray-core/app/dns/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/dns/config.proto package dns diff --git a/xray-core/app/dns/fakedns/fakedns.pb.go b/xray-core/app/dns/fakedns/fakedns.pb.go index 906cb44609..7e3615f2df 100644 --- a/xray-core/app/dns/fakedns/fakedns.pb.go +++ b/xray-core/app/dns/fakedns/fakedns.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/dns/fakedns/fakedns.proto package fakedns diff --git a/xray-core/app/log/command/config.pb.go b/xray-core/app/log/command/config.pb.go index 68f418f91b..c39f466bc8 100644 --- a/xray-core/app/log/command/config.pb.go +++ b/xray-core/app/log/command/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/log/command/config.proto package command diff --git a/xray-core/app/log/command/config_grpc.pb.go b/xray-core/app/log/command/config_grpc.pb.go index 93a695b0a3..3ad1528891 100644 --- a/xray-core/app/log/command/config_grpc.pb.go +++ b/xray-core/app/log/command/config_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.23.1 +// - protoc v5.27.0 // source: app/log/command/config.proto package command diff --git a/xray-core/app/log/config.pb.go b/xray-core/app/log/config.pb.go index 093dd04fbc..7dc40d8006 100644 --- a/xray-core/app/log/config.pb.go +++ b/xray-core/app/log/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/log/config.proto package log diff --git a/xray-core/app/metrics/config.pb.go b/xray-core/app/metrics/config.pb.go index 218a02846d..fd0e6b0a25 100644 --- a/xray-core/app/metrics/config.pb.go +++ b/xray-core/app/metrics/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/metrics/config.proto package metrics diff --git a/xray-core/app/observatory/burst/config.pb.go b/xray-core/app/observatory/burst/config.pb.go index 15a29a5116..bdbcd8c428 100644 --- a/xray-core/app/observatory/burst/config.pb.go +++ b/xray-core/app/observatory/burst/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/observatory/burst/config.proto package burst diff --git a/xray-core/app/observatory/command/command.pb.go b/xray-core/app/observatory/command/command.pb.go index 6f50c245f2..7ec43f3389 100644 --- a/xray-core/app/observatory/command/command.pb.go +++ b/xray-core/app/observatory/command/command.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/observatory/command/command.proto package command diff --git a/xray-core/app/observatory/command/command_grpc.pb.go b/xray-core/app/observatory/command/command_grpc.pb.go index b0a5977982..035cee07aa 100644 --- a/xray-core/app/observatory/command/command_grpc.pb.go +++ b/xray-core/app/observatory/command/command_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.23.1 +// - protoc v5.27.0 // source: app/observatory/command/command.proto package command diff --git a/xray-core/app/observatory/config.pb.go b/xray-core/app/observatory/config.pb.go index 40253497dd..c1ff512e96 100644 --- a/xray-core/app/observatory/config.pb.go +++ b/xray-core/app/observatory/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/observatory/config.proto package observatory diff --git a/xray-core/app/policy/config.pb.go b/xray-core/app/policy/config.pb.go index fefeb4f1bd..adcb363ad2 100644 --- a/xray-core/app/policy/config.pb.go +++ b/xray-core/app/policy/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/policy/config.proto package policy diff --git a/xray-core/app/proxyman/command/command.pb.go b/xray-core/app/proxyman/command/command.pb.go index bc2fb781e8..7ce5904c4a 100644 --- a/xray-core/app/proxyman/command/command.pb.go +++ b/xray-core/app/proxyman/command/command.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/proxyman/command/command.proto package command diff --git a/xray-core/app/proxyman/command/command_grpc.pb.go b/xray-core/app/proxyman/command/command_grpc.pb.go index 0765fc80a1..8478780e89 100644 --- a/xray-core/app/proxyman/command/command_grpc.pb.go +++ b/xray-core/app/proxyman/command/command_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.23.1 +// - protoc v5.27.0 // source: app/proxyman/command/command.proto package command diff --git a/xray-core/app/proxyman/config.pb.go b/xray-core/app/proxyman/config.pb.go index ab08bbea1a..3f4a8fbf74 100644 --- a/xray-core/app/proxyman/config.pb.go +++ b/xray-core/app/proxyman/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/proxyman/config.proto package proxyman diff --git a/xray-core/app/proxyman/inbound/worker.go b/xray-core/app/proxyman/inbound/worker.go index 9a6499f129..539508e19d 100644 --- a/xray-core/app/proxyman/inbound/worker.go +++ b/xray-core/app/proxyman/inbound/worker.go @@ -308,12 +308,11 @@ func (w *udpWorker) callback(b *buf.Buffer, source net.Destination, originalDest sid := session.NewID() ctx = session.ContextWithID(ctx, sid) + outbounds := []*session.Outbound{{}} if originalDest.IsValid() { - outbounds := []*session.Outbound{{ - Target: originalDest, - }} - ctx = session.ContextWithOutbounds(ctx, outbounds) + outbounds[0].Target = originalDest } + ctx = session.ContextWithOutbounds(ctx, outbounds) ctx = session.ContextWithInbound(ctx, &session.Inbound{ Source: source, Gateway: net.UDPDestination(w.address, w.port), diff --git a/xray-core/app/reverse/config.pb.go b/xray-core/app/reverse/config.pb.go index b7899948ef..c7572e2ae0 100644 --- a/xray-core/app/reverse/config.pb.go +++ b/xray-core/app/reverse/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/reverse/config.proto package reverse diff --git a/xray-core/app/router/command/command.pb.go b/xray-core/app/router/command/command.pb.go index 3fc58b2f03..56b5798e31 100644 --- a/xray-core/app/router/command/command.pb.go +++ b/xray-core/app/router/command/command.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/router/command/command.proto package command diff --git a/xray-core/app/router/command/command_grpc.pb.go b/xray-core/app/router/command/command_grpc.pb.go index c64a442304..157fd57ed7 100644 --- a/xray-core/app/router/command/command_grpc.pb.go +++ b/xray-core/app/router/command/command_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.23.1 +// - protoc v5.27.0 // source: app/router/command/command.proto package command diff --git a/xray-core/app/router/config.pb.go b/xray-core/app/router/config.pb.go index d0da6d1414..43f3bcfc82 100644 --- a/xray-core/app/router/config.pb.go +++ b/xray-core/app/router/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/router/config.proto package router diff --git a/xray-core/app/stats/command/command.pb.go b/xray-core/app/stats/command/command.pb.go index cb6a118325..418e7fb1e8 100644 --- a/xray-core/app/stats/command/command.pb.go +++ b/xray-core/app/stats/command/command.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/stats/command/command.proto package command diff --git a/xray-core/app/stats/command/command_grpc.pb.go b/xray-core/app/stats/command/command_grpc.pb.go index cbd024852c..e9420299b7 100644 --- a/xray-core/app/stats/command/command_grpc.pb.go +++ b/xray-core/app/stats/command/command_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.23.1 +// - protoc v5.27.0 // source: app/stats/command/command.proto package command diff --git a/xray-core/app/stats/config.pb.go b/xray-core/app/stats/config.pb.go index 1ca8a4dbef..4f5e04d958 100644 --- a/xray-core/app/stats/config.pb.go +++ b/xray-core/app/stats/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: app/stats/config.proto package stats diff --git a/xray-core/common/log/log.pb.go b/xray-core/common/log/log.pb.go index 7d3c45bdf3..2a970b546b 100644 --- a/xray-core/common/log/log.pb.go +++ b/xray-core/common/log/log.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/log/log.proto package log diff --git a/xray-core/common/net/address.pb.go b/xray-core/common/net/address.pb.go index feb9f60623..fd61b16aea 100644 --- a/xray-core/common/net/address.pb.go +++ b/xray-core/common/net/address.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/net/address.proto package net diff --git a/xray-core/common/net/destination.pb.go b/xray-core/common/net/destination.pb.go index 06afca473d..3e4de52cbd 100644 --- a/xray-core/common/net/destination.pb.go +++ b/xray-core/common/net/destination.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/net/destination.proto package net diff --git a/xray-core/common/net/network.pb.go b/xray-core/common/net/network.pb.go index 29f487e4cc..f720ca545c 100644 --- a/xray-core/common/net/network.pb.go +++ b/xray-core/common/net/network.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/net/network.proto package net diff --git a/xray-core/common/net/port.pb.go b/xray-core/common/net/port.pb.go index 245e513197..36af494210 100644 --- a/xray-core/common/net/port.pb.go +++ b/xray-core/common/net/port.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/net/port.proto package net diff --git a/xray-core/common/protocol/headers.pb.go b/xray-core/common/protocol/headers.pb.go index 2aaddcef67..bb6e810b4d 100644 --- a/xray-core/common/protocol/headers.pb.go +++ b/xray-core/common/protocol/headers.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/protocol/headers.proto package protocol diff --git a/xray-core/common/protocol/server_spec.pb.go b/xray-core/common/protocol/server_spec.pb.go index cbb083dfa2..cb19e91ec0 100644 --- a/xray-core/common/protocol/server_spec.pb.go +++ b/xray-core/common/protocol/server_spec.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/protocol/server_spec.proto package protocol diff --git a/xray-core/common/protocol/user.pb.go b/xray-core/common/protocol/user.pb.go index cb1c340f33..0230b37c5a 100644 --- a/xray-core/common/protocol/user.pb.go +++ b/xray-core/common/protocol/user.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/protocol/user.proto package protocol diff --git a/xray-core/common/serial/typed_message.pb.go b/xray-core/common/serial/typed_message.pb.go index bb34432885..719a2d04d6 100644 --- a/xray-core/common/serial/typed_message.pb.go +++ b/xray-core/common/serial/typed_message.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: common/serial/typed_message.proto package serial diff --git a/xray-core/core/config.pb.go b/xray-core/core/config.pb.go index 0d5b1ea899..1b2591dc27 100644 --- a/xray-core/core/config.pb.go +++ b/xray-core/core/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: core/config.proto package core diff --git a/xray-core/infra/vprotogen/main.go b/xray-core/infra/vprotogen/main.go index 10aaa25445..f62ae4aec5 100644 --- a/xray-core/infra/vprotogen/main.go +++ b/xray-core/infra/vprotogen/main.go @@ -174,11 +174,14 @@ func main() { suffix = ".exe" } - targetedVersion, err := getProjectProtocVersion("https://raw.githubusercontent.com/xtls/xray-core/HEAD/core/config.pb.go") - if err != nil { - fmt.Println(err) - os.Exit(1) - } + /* + targetedVersion, err := getProjectProtocVersion("https://raw.githubusercontent.com/XTLS/Xray-core/HEAD/core/config.pb.go") + if err != nil { + fmt.Println(err) + os.Exit(1) + } + */ + targetedVersion := "" protoc, err := whichProtoc(suffix, targetedVersion) if err != nil { diff --git a/xray-core/proxy/blackhole/config.pb.go b/xray-core/proxy/blackhole/config.pb.go index 38a0f76d6c..f2e542937d 100644 --- a/xray-core/proxy/blackhole/config.pb.go +++ b/xray-core/proxy/blackhole/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/blackhole/config.proto package blackhole diff --git a/xray-core/proxy/dns/config.pb.go b/xray-core/proxy/dns/config.pb.go index af9d7913c6..d8a46aed7c 100644 --- a/xray-core/proxy/dns/config.pb.go +++ b/xray-core/proxy/dns/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/dns/config.proto package dns diff --git a/xray-core/proxy/dokodemo/config.pb.go b/xray-core/proxy/dokodemo/config.pb.go index 7dfd65686d..4f698be153 100644 --- a/xray-core/proxy/dokodemo/config.pb.go +++ b/xray-core/proxy/dokodemo/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/dokodemo/config.proto package dokodemo diff --git a/xray-core/proxy/freedom/config.pb.go b/xray-core/proxy/freedom/config.pb.go index 5db2148d5e..c8f6c3df98 100644 --- a/xray-core/proxy/freedom/config.pb.go +++ b/xray-core/proxy/freedom/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/freedom/config.proto package freedom diff --git a/xray-core/proxy/http/config.pb.go b/xray-core/proxy/http/config.pb.go index 023c1b1b6b..c4494a26f1 100644 --- a/xray-core/proxy/http/config.pb.go +++ b/xray-core/proxy/http/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/http/config.proto package http diff --git a/xray-core/proxy/loopback/config.pb.go b/xray-core/proxy/loopback/config.pb.go index 53123ca1bb..f66696ee40 100644 --- a/xray-core/proxy/loopback/config.pb.go +++ b/xray-core/proxy/loopback/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/loopback/config.proto package loopback diff --git a/xray-core/proxy/shadowsocks/config.pb.go b/xray-core/proxy/shadowsocks/config.pb.go index 6c2d0ddb56..03dd33421e 100644 --- a/xray-core/proxy/shadowsocks/config.pb.go +++ b/xray-core/proxy/shadowsocks/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/shadowsocks/config.proto package shadowsocks diff --git a/xray-core/proxy/shadowsocks_2022/config.pb.go b/xray-core/proxy/shadowsocks_2022/config.pb.go index 07f459a014..41978474be 100644 --- a/xray-core/proxy/shadowsocks_2022/config.pb.go +++ b/xray-core/proxy/shadowsocks_2022/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/shadowsocks_2022/config.proto package shadowsocks_2022 diff --git a/xray-core/proxy/socks/config.pb.go b/xray-core/proxy/socks/config.pb.go index b10affd351..6471babfdb 100644 --- a/xray-core/proxy/socks/config.pb.go +++ b/xray-core/proxy/socks/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/socks/config.proto package socks diff --git a/xray-core/proxy/trojan/config.pb.go b/xray-core/proxy/trojan/config.pb.go index 29a307eab9..6cd360ede2 100644 --- a/xray-core/proxy/trojan/config.pb.go +++ b/xray-core/proxy/trojan/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/trojan/config.proto package trojan diff --git a/xray-core/proxy/vless/account.pb.go b/xray-core/proxy/vless/account.pb.go index 203e3376bc..24564a3630 100644 --- a/xray-core/proxy/vless/account.pb.go +++ b/xray-core/proxy/vless/account.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/vless/account.proto package vless diff --git a/xray-core/proxy/vless/encoding/addons.pb.go b/xray-core/proxy/vless/encoding/addons.pb.go index b6a2925d5d..c876d543a8 100644 --- a/xray-core/proxy/vless/encoding/addons.pb.go +++ b/xray-core/proxy/vless/encoding/addons.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/vless/encoding/addons.proto package encoding diff --git a/xray-core/proxy/vless/inbound/config.pb.go b/xray-core/proxy/vless/inbound/config.pb.go index 9a763b1dc2..c60cd9c5c6 100644 --- a/xray-core/proxy/vless/inbound/config.pb.go +++ b/xray-core/proxy/vless/inbound/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/vless/inbound/config.proto package inbound diff --git a/xray-core/proxy/vless/outbound/config.pb.go b/xray-core/proxy/vless/outbound/config.pb.go index 95955b1e1c..f7fa0e3314 100644 --- a/xray-core/proxy/vless/outbound/config.pb.go +++ b/xray-core/proxy/vless/outbound/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/vless/outbound/config.proto package outbound diff --git a/xray-core/proxy/vmess/account.pb.go b/xray-core/proxy/vmess/account.pb.go index ef212c387e..d961130a04 100644 --- a/xray-core/proxy/vmess/account.pb.go +++ b/xray-core/proxy/vmess/account.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/vmess/account.proto package vmess diff --git a/xray-core/proxy/vmess/inbound/config.pb.go b/xray-core/proxy/vmess/inbound/config.pb.go index 0db3fdc191..5b0c8c984c 100644 --- a/xray-core/proxy/vmess/inbound/config.pb.go +++ b/xray-core/proxy/vmess/inbound/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/vmess/inbound/config.proto package inbound diff --git a/xray-core/proxy/vmess/outbound/config.pb.go b/xray-core/proxy/vmess/outbound/config.pb.go index 7894241e42..ecadd1c035 100644 --- a/xray-core/proxy/vmess/outbound/config.pb.go +++ b/xray-core/proxy/vmess/outbound/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/vmess/outbound/config.proto package outbound diff --git a/xray-core/proxy/wireguard/config.pb.go b/xray-core/proxy/wireguard/config.pb.go index 3563685e03..25036ba7d6 100644 --- a/xray-core/proxy/wireguard/config.pb.go +++ b/xray-core/proxy/wireguard/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: proxy/wireguard/config.proto package wireguard diff --git a/xray-core/transport/global/config.pb.go b/xray-core/transport/global/config.pb.go index f77b1da343..7459ab6bb5 100644 --- a/xray-core/transport/global/config.pb.go +++ b/xray-core/transport/global/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/global/config.proto package global diff --git a/xray-core/transport/internet/config.pb.go b/xray-core/transport/internet/config.pb.go index 629b03152c..e262a1a81b 100644 --- a/xray-core/transport/internet/config.pb.go +++ b/xray-core/transport/internet/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/config.proto package internet diff --git a/xray-core/transport/internet/domainsocket/config.pb.go b/xray-core/transport/internet/domainsocket/config.pb.go index 50bf09841e..3ab1adc6d4 100644 --- a/xray-core/transport/internet/domainsocket/config.pb.go +++ b/xray-core/transport/internet/domainsocket/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/domainsocket/config.proto package domainsocket diff --git a/xray-core/transport/internet/grpc/config.pb.go b/xray-core/transport/internet/grpc/config.pb.go index 6c441171bf..a0996e57b8 100644 --- a/xray-core/transport/internet/grpc/config.pb.go +++ b/xray-core/transport/internet/grpc/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/grpc/config.proto package grpc diff --git a/xray-core/transport/internet/grpc/encoding/stream.pb.go b/xray-core/transport/internet/grpc/encoding/stream.pb.go index cdab47cd06..519adb7e82 100644 --- a/xray-core/transport/internet/grpc/encoding/stream.pb.go +++ b/xray-core/transport/internet/grpc/encoding/stream.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/grpc/encoding/stream.proto package encoding diff --git a/xray-core/transport/internet/grpc/encoding/stream_grpc.pb.go b/xray-core/transport/internet/grpc/encoding/stream_grpc.pb.go index e7cc8a5e0c..5daf42a675 100644 --- a/xray-core/transport/internet/grpc/encoding/stream_grpc.pb.go +++ b/xray-core/transport/internet/grpc/encoding/stream_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 -// - protoc v4.23.1 +// - protoc v5.27.0 // source: transport/internet/grpc/encoding/stream.proto package encoding diff --git a/xray-core/transport/internet/headers/dns/config.pb.go b/xray-core/transport/internet/headers/dns/config.pb.go index 418bedadc4..78e5e99b8c 100644 --- a/xray-core/transport/internet/headers/dns/config.pb.go +++ b/xray-core/transport/internet/headers/dns/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/headers/dns/config.proto package dns diff --git a/xray-core/transport/internet/headers/http/config.pb.go b/xray-core/transport/internet/headers/http/config.pb.go index f7c7109dee..681323f14a 100644 --- a/xray-core/transport/internet/headers/http/config.pb.go +++ b/xray-core/transport/internet/headers/http/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/headers/http/config.proto package http diff --git a/xray-core/transport/internet/headers/noop/config.pb.go b/xray-core/transport/internet/headers/noop/config.pb.go index 8a40604cc0..844c3f2b36 100644 --- a/xray-core/transport/internet/headers/noop/config.pb.go +++ b/xray-core/transport/internet/headers/noop/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/headers/noop/config.proto package noop diff --git a/xray-core/transport/internet/headers/srtp/config.pb.go b/xray-core/transport/internet/headers/srtp/config.pb.go index 7dd2d15d49..f0bff8f44d 100644 --- a/xray-core/transport/internet/headers/srtp/config.pb.go +++ b/xray-core/transport/internet/headers/srtp/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/headers/srtp/config.proto package srtp diff --git a/xray-core/transport/internet/headers/tls/config.pb.go b/xray-core/transport/internet/headers/tls/config.pb.go index 864b2042f6..3c4786ce1c 100644 --- a/xray-core/transport/internet/headers/tls/config.pb.go +++ b/xray-core/transport/internet/headers/tls/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/headers/tls/config.proto package tls diff --git a/xray-core/transport/internet/headers/utp/config.pb.go b/xray-core/transport/internet/headers/utp/config.pb.go index 057a8bfd56..ed04788039 100644 --- a/xray-core/transport/internet/headers/utp/config.pb.go +++ b/xray-core/transport/internet/headers/utp/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/headers/utp/config.proto package utp diff --git a/xray-core/transport/internet/headers/wechat/config.pb.go b/xray-core/transport/internet/headers/wechat/config.pb.go index 38d00a0c24..3961181031 100644 --- a/xray-core/transport/internet/headers/wechat/config.pb.go +++ b/xray-core/transport/internet/headers/wechat/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/headers/wechat/config.proto package wechat diff --git a/xray-core/transport/internet/headers/wireguard/config.pb.go b/xray-core/transport/internet/headers/wireguard/config.pb.go index efe7f27c1b..2212c69ec5 100644 --- a/xray-core/transport/internet/headers/wireguard/config.pb.go +++ b/xray-core/transport/internet/headers/wireguard/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/headers/wireguard/config.proto package wireguard diff --git a/xray-core/transport/internet/http/config.pb.go b/xray-core/transport/internet/http/config.pb.go index cbf1a3b7da..d0f7e63265 100644 --- a/xray-core/transport/internet/http/config.pb.go +++ b/xray-core/transport/internet/http/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/http/config.proto package http diff --git a/xray-core/transport/internet/httpupgrade/config.pb.go b/xray-core/transport/internet/httpupgrade/config.pb.go index a4441fdec8..9c0a4dee75 100644 --- a/xray-core/transport/internet/httpupgrade/config.pb.go +++ b/xray-core/transport/internet/httpupgrade/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/httpupgrade/config.proto package httpupgrade diff --git a/xray-core/transport/internet/kcp/config.pb.go b/xray-core/transport/internet/kcp/config.pb.go index 213b904203..d4273da537 100644 --- a/xray-core/transport/internet/kcp/config.pb.go +++ b/xray-core/transport/internet/kcp/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/kcp/config.proto package kcp diff --git a/xray-core/transport/internet/quic/config.pb.go b/xray-core/transport/internet/quic/config.pb.go index 76a0752b66..0a387a101f 100644 --- a/xray-core/transport/internet/quic/config.pb.go +++ b/xray-core/transport/internet/quic/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/quic/config.proto package quic diff --git a/xray-core/transport/internet/reality/config.pb.go b/xray-core/transport/internet/reality/config.pb.go index 5c6a5d3fbe..99dec21452 100644 --- a/xray-core/transport/internet/reality/config.pb.go +++ b/xray-core/transport/internet/reality/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/reality/config.proto package reality diff --git a/xray-core/transport/internet/tcp/config.pb.go b/xray-core/transport/internet/tcp/config.pb.go index 2d1eb352bb..e06c481bf0 100644 --- a/xray-core/transport/internet/tcp/config.pb.go +++ b/xray-core/transport/internet/tcp/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/tcp/config.proto package tcp diff --git a/xray-core/transport/internet/tls/config.pb.go b/xray-core/transport/internet/tls/config.pb.go index d8c9a423a1..2613c3b952 100644 --- a/xray-core/transport/internet/tls/config.pb.go +++ b/xray-core/transport/internet/tls/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/tls/config.proto package tls diff --git a/xray-core/transport/internet/udp/config.pb.go b/xray-core/transport/internet/udp/config.pb.go index 6b0c2079da..760e230c5a 100644 --- a/xray-core/transport/internet/udp/config.pb.go +++ b/xray-core/transport/internet/udp/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/udp/config.proto package udp diff --git a/xray-core/transport/internet/websocket/config.pb.go b/xray-core/transport/internet/websocket/config.pb.go index 9fdbcbaf9d..9631e7e912 100644 --- a/xray-core/transport/internet/websocket/config.pb.go +++ b/xray-core/transport/internet/websocket/config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 -// protoc v4.23.1 +// protoc-gen-go v1.34.1 +// protoc v5.27.0 // source: transport/internet/websocket/config.proto package websocket diff --git a/yass/.circleci/config.yml b/yass/.circleci/config.yml index 4a6c383e8a..c50a34a215 100644 --- a/yass/.circleci/config.yml +++ b/yass/.circleci/config.yml @@ -170,6 +170,8 @@ commands: # test SOCKS5 without auth ./build/yass_test --gtest_filter="Ss*/SOCKS5" --username '' --password '' ./build/yass_test --gtest_filter="Ss*/SOCKS5H" --username '' --password '' + # test HTTPS/HTTP2 without auth + ./build/yass_test --gtest_filter="Ss*/HTTP*" --username '' --password '' # somehow inside the container DNS hosts file resolves localhost to 127.0.0.1 only not ::1 # due to the incompleteness grep localhost /etc/hosts diff --git a/yass/CMakeLists.txt b/yass/CMakeLists.txt index 064897286d..1881859493 100644 --- a/yass/CMakeLists.txt +++ b/yass/CMakeLists.txt @@ -2711,6 +2711,21 @@ if (BORINGSSL_BUILD_TESTS) USES_TERMINAL) endif() +# ***************************************************************************************** +# modp_b64 Library +# ***************************************************************************************** +add_library(modp_b64 STATIC + third_party/modp_b64/modp_b64.h + third_party/modp_b64/modp_b64_data.h + third_party/modp_b64/modp_b64.cc +) +target_include_directories(modp_b64 PUBLIC + third_party/modp_b64) + +target_compile_options(modp_b64 PRIVATE "-Wno-error") + +set(SUPPORT_LIBS modp_b64 ${SUPPORT_LIBS}) + # ***************************************************************************************** # mbedssl Library # ***************************************************************************************** @@ -4647,6 +4662,7 @@ if(NOT CMAKE_SKIP_INSTALL_RULES) file(READ third_party/libunwind/trunk/LICENSE.TXT _LIBUNWIND_LICENSE) file(READ third_party/lss/LICENSE _LSS_LICENSE) file(READ third_party/mbedtls/LICENSE _MBEDTLS_LICENSE) + file(READ third_party/modp_b64/LICENSE _MODP_B64_LICENSE) file(READ third_party/protobuf/LICENSE _PROTOBUF_LICENSE) file(READ third_party/quiche/src/LICENSE _QUICHE_LICENSE) file(READ third_party/re2/LICENSE _RE2_LICENSE) diff --git a/yass/LICENSE.cmake.in b/yass/LICENSE.cmake.in index 35a9f46389..e1e78d6a96 100644 --- a/yass/LICENSE.cmake.in +++ b/yass/LICENSE.cmake.in @@ -44,6 +44,10 @@ @_MBEDTLS_LICENSE@ +=== modp_b64 == + +@_MODP_B64_LICENSE@ + === protobuf == @_PROTOBUF_LICENSE@ diff --git a/yass/README.md b/yass/README.md index b655a8ecea..a1a99e7bff 100644 --- a/yass/README.md +++ b/yass/README.md @@ -28,17 +28,17 @@ Post Quantum Kyber Support (not enabled by default) is added on all of supported See [Protecting Chrome Traffic with Hybrid Kyber KEM](https://blog.chromium.org/2023/08/protecting-chrome-traffic-with-hybrid.html) for more. ### Prebuilt binaries -- Android [download apk](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass-android-release-arm64-1.10.2.apk) or [download 32-bit apk](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass-android-release-arm-1.10.2.apk) +- Android [download apk](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass-android-release-arm64-1.10.3.apk) or [download 32-bit apk](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass-android-release-arm-1.10.3.apk) - iOS [join via TestFlight](https://testflight.apple.com/join/6AkiEq09) -- Windows [download installer](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass-mingw-win7-release-x86_64-1.10.2-system-installer.exe) or [download 32-bit installer](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass-mingw-winxp-release-i686-1.10.2-system-installer.exe) [(require vc 2010 runtime)][vs2010_x86] or [download woa arm64 installer](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass-mingw-release-aarch64-1.10.2-system-installer.exe) -- macOS [download intel dmg](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass-macos-release-x64-1.10.2.dmg) or [download apple silicon dmg](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass-macos-release-arm64-1.10.2.dmg) +- Windows [download installer](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass-mingw-win7-release-x86_64-1.10.3-system-installer.exe) or [download 32-bit installer](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass-mingw-winxp-release-i686-1.10.3-system-installer.exe) [(require vc 2010 runtime)][vs2010_x86] or [download woa arm64 installer](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass-mingw-release-aarch64-1.10.3-system-installer.exe) +- macOS [download intel dmg](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass-macos-release-x64-1.10.3.dmg) or [download apple silicon dmg](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass-macos-release-arm64-1.10.3.dmg) > via homebrew: `brew install --cask yass` -- Linux [download rpm](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass.el7.x86_64.1.10.2-0.rpm) or [download deb](https://github.com/Chilledheart/yass/releases/download/1.10.2/yass-client-ubuntu-16.04-xenial_amd64.1.10.2.deb) +- Linux [download rpm](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass.el7.x86_64.1.10.3.rpm) or [download deb](https://github.com/Chilledheart/yass/releases/download/1.10.3/yass-ubuntu-16.04-xenial_amd64.1.10.3.deb) -View more at [Release Page](https://github.com/Chilledheart/yass/releases/tag/1.10.2) +View more at [Release Page](https://github.com/Chilledheart/yass/releases/tag/1.10.3) ### NaïveProxy-Compatible Protocol Support -Cipher http2 is NaïveProxy-compatible. +Cipher http2 and https are NaïveProxy-compatible. See [NaïveProxy](https://github.com/klzgrad/naiveproxy)'s project homepage for support. @@ -46,7 +46,7 @@ See [NaïveProxy](https://github.com/klzgrad/naiveproxy)'s project homepage for Visit wiki's [Usages](https://github.com/Chilledheart/yass/wiki/Usage). ## Build from Source -Take a look at [more instructions](BUILDING.md). +Take a look at [instructions](BUILDING.md). ## Sponsor Me Please visit [the pages site](https://letshack.info). @@ -74,27 +74,10 @@ Please visit [the pages site](https://letshack.info). ## Additional Features -### Rate Limit (only on CLI) -Pass `--limit_rate rate` to command line. -Limits the _rate_ of response transmission to a client. Uint can be `(none)`, `k` and `m`. - -### Change of Congestion Algorithm (Linux only) -Pass `--congestion_algorithm algo` to command line. -Specify _algo_ as TCP congestion control algorithm for underlying TCP connections. - -### Allow custom CA (only on CLI, client only) -Pass `--certificate_chain_file file` to command line. -Use custom certificate chain provided by _file_ to verify server's certificate. - -### Server Side Support -All ciphers supported by client are also supported by `yass_server`. Read more from manpage _yass_server(1)_ - -See [Server Usage](https://github.com/Chilledheart/yass/wiki/Usage:-server-setup) for more. - -### Experimental SOCKS cipher Support +### SOCKS cipher Support Experimental socks4/socks4a/socks5/socks5h cipher support is added for both of CLI and GUI. -### Experimental DoH (DNS over HTTPS) and DoT (DNS over TLS) Support +### DoH (DNS over HTTPS) and DoT (DNS over TLS) Support Experimental DoH and DoT support is added for both of CLI and GUI. ### Supplementary Support for ISRG Root X2 and ISRG Root X1 ca which is missing on some machines @@ -103,6 +86,25 @@ These ca certificates are provided in both builtin ca bundle support and supplem ### Supplementary Support for DigiCert Global Root G2 ca which is missing on some machines These ca certificates are provided in both builtin ca bundle support and supplementary ca bundle support (bundled). +### Specify Rate Limit (Command Line only) +Pass `--limit_rate rate` to command line. +Limits the _rate_ of response transmission to a client. +Uint can be `(none)`, `k` and `m`. + +### Specify TCP Congestion Algorithm (Command Line only) +Pass `--congestion_algorithm algo` to command line. +Specify _algo_ as TCP congestion control algorithm for underlying TCP connections (Linux only). +See more at manpage [_tcp(7)_](https://linux.die.net/man/7/tcp) + +### Use custom CA (Command Line only) +Pass `--certificate_chain_file file` to command line. +Use custom certificate chain provided by _file_ to verify server's certificate. + +### Use server Side Support (Commmand Line only) +All ciphers supported by client are also supported by `yass_server`. +See more at manpage _yass_server(1)_ + +See [Server Usage](https://github.com/Chilledheart/yass/wiki/Usage:-server-setup) for more. + [license-link]: LICENSE [vs2010_x86]: https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe - diff --git a/yass/debian/changelog b/yass/debian/changelog index eb77323602..ce7c2c7517 100644 --- a/yass/debian/changelog +++ b/yass/debian/changelog @@ -1,3 +1,8 @@ +yass (1.10.3-1) UNRELEASED; urgency=medium + + * net: support https protocol via caddy. + + -- Chilledheart Sun, 26 May 2024 14:48:28 +0800 yass (1.10.2-1) UNRELEASED; urgency=medium * net: reduce memory footprint peak and cpu peak. diff --git a/yass/src/cli/cli_connection.cpp b/yass/src/cli/cli_connection.cpp index fd23c1d0a4..29cf709cca 100644 --- a/yass/src/cli/cli_connection.cpp +++ b/yass/src/cli/cli_connection.cpp @@ -48,10 +48,9 @@ static std::vector GenerateHeaders(std::vector(const_cast(reinterpret_cast(user_pass.c_str())), user_pass.size())); } static bool g_nonindex_codes_initialized; @@ -1035,12 +1034,18 @@ try_again: } else #endif if (upstream_https_fallback_) { - if (upstream_handshake_) { + if (upstream_https_handshake_) { ReadUpstreamHttpsHandshake(buf, ec); if (ec) { return nullptr; } } + if (upstream_https_chunked_) { + ReadUpstreamHttpsChunk(buf, ec); + if (ec) { + return nullptr; + } + } downstream_.push_back(buf); } else { if (socks5_method_select_handshake_) { @@ -1089,9 +1094,9 @@ out: } void CliConnection::ReadUpstreamHttpsHandshake(std::shared_ptr buf, asio::error_code& ec) { - DCHECK(upstream_handshake_); + DCHECK(upstream_https_handshake_); - upstream_handshake_ = false; + upstream_https_handshake_ = false; HttpResponseParser parser; bool ok; @@ -1104,6 +1109,10 @@ void CliConnection::ReadUpstreamHttpsHandshake(std::shared_ptr buf, asio: if (ok && parser.status_code() == 200) { buf->trimStart(nparsed); buf->retreat(nparsed); + if (parser.transfer_encoding_is_chunked()) { + upstream_https_chunked_ = true; + VLOG(1) << "Connection (client) " << connection_id() << " upstream http chunked encoding"; + } } else { if (!ok) { LOG(WARNING) << "Connection (client) " << connection_id() @@ -1122,6 +1131,44 @@ void CliConnection::ReadUpstreamHttpsHandshake(std::shared_ptr buf, asio: } } +void CliConnection::ReadUpstreamHttpsChunk(std::shared_ptr buf, asio::error_code& ec) { + DCHECK(upstream_https_chunked_); + + HttpResponseParser parser; + + bool ok; + int nparsed = parser.Parse(buf, &ok); + + if (nparsed) { + VLOG(3) << "Connection (client) " << connection_id() + << " chunked http: " << std::string(reinterpret_cast(buf->data()), nparsed); + } + if (ok && parser.status_code() == 200) { + buf->trimStart(nparsed); + buf->retreat(nparsed); + upstream_https_chunked_ = false; + if (parser.content_length() != 0) { + LOG(WARNING) << "Connection (client) " << connection_id() << " upstream server returns unexpected body"; + ec = asio::error::invalid_argument; + return; + } + if (buf->empty()) { + ec = asio::error::try_again; + return; + } + } else { + if (!ok) { + LOG(WARNING) << "Connection (client) " << connection_id() + << " upstream server unhandled: " << parser.ErrorMessage() << ": " + << std::string(reinterpret_cast(buf->data()), nparsed); + } else { + LOG(WARNING) << "Connection (client) " << connection_id() << " upstream server returns: " << parser.status_code(); + } + ec = asio::error::connection_refused; + return; + } +} + void CliConnection::WriteUpstreamMethodSelectRequest() { socks5::method_select_request_header method_select_header; method_select_header.ver = socks5::version; @@ -2122,7 +2169,10 @@ void CliConnection::connected() { // authority = [ userinfo "@" ] host [ ":" port ] headers.emplace_back(":authority"s, hostname_and_port); headers.emplace_back("host"s, hostname_and_port); - headers.emplace_back("proxy-authorization"s, absl::StrCat("basic ", GetProxyAuthorizationIdentity())); + bool auth_required = !absl::GetFlag(FLAGS_username).empty() && !absl::GetFlag(FLAGS_password).empty(); + if (auth_required) { + headers.emplace_back("proxy-authorization"s, absl::StrCat("basic ", GetProxyAuthorizationIdentity())); + } // Send "Padding" header // originated from naive_proxy_delegate.go;func ServeHTTP if (padding_support_) { @@ -2163,12 +2213,23 @@ void CliConnection::connected() { hostname_and_port = absl::StrCat("[", host, "]", ":", port); } + bool auth_required = !absl::GetFlag(FLAGS_username).empty() && !absl::GetFlag(FLAGS_password).empty(); + std::string hdr = absl::StrFormat( "CONNECT %s HTTP/1.1\r\n" "Host: %s\r\n" + "Proxy-Authorization: %s\r\n" "Proxy-Connection: Close\r\n" "\r\n", - hostname_and_port.c_str(), hostname_and_port.c_str()); + hostname_and_port.c_str(), hostname_and_port.c_str(), absl::StrCat("basic ", GetProxyAuthorizationIdentity())); + if (!auth_required) { + hdr = absl::StrFormat( + "CONNECT %s HTTP/1.1\r\n" + "Host: %s\r\n" + "Proxy-Connection: Close\r\n" + "\r\n", + hostname_and_port.c_str(), hostname_and_port.c_str()); + } // write variable address directly as https header upstream_.push_back(hdr.data(), hdr.size()); } else { diff --git a/yass/src/cli/cli_connection.hpp b/yass/src/cli/cli_connection.hpp index edca0ea694..9e7e55c0b1 100644 --- a/yass/src/cli/cli_connection.hpp +++ b/yass/src/cli/cli_connection.hpp @@ -366,6 +366,7 @@ class CliConnection : public RefCountedThreadSafe, private: void ReadUpstreamHttpsHandshake(std::shared_ptr buf, asio::error_code& ec); + void ReadUpstreamHttpsChunk(std::shared_ptr buf, asio::error_code& ec); void WriteUpstreamMethodSelectRequest(); void ReadUpstreamMethodSelectResponse(std::shared_ptr buf, asio::error_code& ec); @@ -377,7 +378,9 @@ class CliConnection : public RefCountedThreadSafe, void ReadUpstreamSocksResponse(std::shared_ptr buf, asio::error_code& ec); /// the state of https fallback handshake (upstream) - bool upstream_handshake_ = true; + bool upstream_https_handshake_ = true; + /// the state of https chunked (upstream) + bool upstream_https_chunked_ = false; /// the state of socks5 method select handshake (upstream) bool socks5_method_select_handshake_ = false; bool socks5_auth_handshake_ = false; diff --git a/yass/src/config/config.cpp b/yass/src/config/config.cpp index ddda2fab36..e96116dbab 100644 --- a/yass/src/config/config.cpp +++ b/yass/src/config/config.cpp @@ -190,6 +190,12 @@ std::string ReadConfigFromArgument(std::string_view server_host, } } + if (CIPHER_METHOD_IS_HTTP(method)) { + if (username.empty() ^ password.empty()) { + err_msg << ",HTTP requires both of username and passsword"; + } + } + if (local_host.empty() || local_host.size() >= TLSEXT_MAXLEN_host_name) { err_msg << ",Invalid Local Host: " << local_host; } @@ -282,6 +288,12 @@ std::string ReadConfigFromArgument(std::string_view server_host, } } + if (CIPHER_METHOD_IS_HTTP(method)) { + if (username.empty() ^ password.empty()) { + err_msg << ",HTTP requires both of username and passsword"; + } + } + if (local_host.empty() || local_host.size() >= TLSEXT_MAXLEN_host_name) { err_msg << ",Invalid Local Host: " << local_host; } diff --git a/yass/src/crypto/crypter_export.hpp b/yass/src/crypto/crypter_export.hpp index 1b7d24b928..55c39e97b1 100644 --- a/yass/src/crypto/crypter_export.hpp +++ b/yass/src/crypto/crypter_export.hpp @@ -60,10 +60,12 @@ #define CIPHER_METHOD_IS_HTTP2(m) ((m) == CRYPTO_HTTP2_PLAINTEXT || (m) == CRYPTO_HTTP2) #define CIPHER_METHOD_IS_TLS(m) ((m) == CRYPTO_HTTPS || (m) == CRYPTO_HTTP2) #define CIPHER_METHOD_IS_HTTPS_FALLBACK(m) ((m) == CRYPTO_HTTPS) +#define CIPHER_METHOD_IS_HTTP(m) ((m) == CRYPTO_HTTPS || (m) == CRYPTO_HTTP2_PLAINTEXT || (m) == CRYPTO_HTTP2) #else #define CIPHER_METHOD_IS_HTTP2(m) false #define CIPHER_METHOD_IS_TLS(m) ((m) == CRYPTO_HTTPS) #define CIPHER_METHOD_IS_HTTPS_FALLBACK(m) ((m) == CRYPTO_HTTPS) +#define CIPHER_METHOD_IS_HTTP(m) ((m) == CRYPTO_HTTPS) #endif #define CIPHER_METHOD_MAP_SOCKS(XX) \ diff --git a/yass/src/net/base64.cpp b/yass/src/net/base64.cpp index f73228bd02..d265a2363f 100644 --- a/yass/src/net/base64.cpp +++ b/yass/src/net/base64.cpp @@ -3,40 +3,75 @@ #include "net/base64.hpp" +#include #include +#include "core/logging.hpp" -#include "third_party/boringssl/src/include/openssl/base64.h" +#include "third_party/modp_b64/modp_b64.h" + +using namespace gurl_base; namespace net { -std::string Base64Encode(const uint8_t* data, size_t length) { +namespace { + +ModpDecodePolicy GetModpPolicy(Base64DecodePolicy policy) { + switch (policy) { + case Base64DecodePolicy::kStrict: + return ModpDecodePolicy::kStrict; + case Base64DecodePolicy::kForgiving: + return ModpDecodePolicy::kForgiving; + } + DCHECK(false) << "Unexpect Base64 Decode Policy: " << (int)policy; + return ModpDecodePolicy::kStrict; +} + +} // namespace + +std::string Base64Encode(absl::Span input) { std::string output; - size_t out_len; - EVP_EncodedLength(&out_len, length); // makes room for null byte - output.resize(out_len); - - // modp_b64_encode_len() returns at least 1, so output[0] is safe to use. - const size_t output_size = EVP_EncodeBlock(reinterpret_cast(&(output[0])), data, length); - - output.resize(output_size); + Base64EncodeAppend(input, &output); return output; } -void Base64Encode(std::string_view input, std::string* output) { - *output = Base64Encode(reinterpret_cast(&*input.begin()), input.size()); +void Base64EncodeAppend(absl::Span input, std::string* output) { + // Ensure `modp_b64_encode_data_len` will not overflow. + CHECK_LE(input.size(), MODP_B64_MAX_INPUT_LEN); + size_t encode_data_len = modp_b64_encode_data_len(input.size()); + + size_t prefix_len = output->size(); + output->resize(encode_data_len + prefix_len); + + const size_t output_size = + modp_b64_encode_data(output->data() + prefix_len, reinterpret_cast(input.data()), input.size()); + CHECK_EQ(output->size(), prefix_len + output_size); } -bool Base64Decode(std::string_view input, std::string* output) { +bool Base64Decode(std::string_view input, std::string* output, Base64DecodePolicy policy) { std::string temp; - size_t out_len; - EVP_DecodedLength(&out_len, input.size()); - temp.resize(out_len); + temp.resize(modp_b64_decode_len(input.size())); // does not null terminate result since result is binary data! size_t input_size = input.size(); - int output_size = EVP_DecodeBase64(reinterpret_cast(&(temp[0])), &out_len, out_len, - reinterpret_cast(input.data()), input_size); - if (output_size <= 0) + size_t output_size = modp_b64_decode(&(temp[0]), input.data(), input_size, GetModpPolicy(policy)); + + // Forgiving mode requires whitespace to be stripped prior to decoding. + // We don't do that in the above code to ensure that the "happy path" of + // input without whitespace is as fast as possible. Since whitespace in input + // will always cause `modp_b64_decode` to fail, just handle whitespace + // stripping on failure. This is not much slower than just scanning for + // whitespace first, even for input with whitespace. + if (output_size == MODP_B64_ERROR && policy == Base64DecodePolicy::kForgiving) { + // We could use `output` here to avoid an allocation when decoding is done + // in-place, but it violates the API contract that `output` is only modified + // on success. + std::string input_without_whitespace; + RemoveChars(input, kInfraAsciiWhitespace, &input_without_whitespace); + output_size = modp_b64_decode(&(temp[0]), input_without_whitespace.data(), input_without_whitespace.size(), + GetModpPolicy(policy)); + } + + if (output_size == MODP_B64_ERROR) return false; temp.resize(output_size); @@ -44,17 +79,13 @@ bool Base64Decode(std::string_view input, std::string* output) { return true; } -absl::optional> Base64Decode(std::string_view input) { - std::vector ret; - size_t out_len; - EVP_DecodedLength(&out_len, input.size()); - ret.resize(out_len); +std::optional> Base64Decode(std::string_view input) { + std::vector ret(modp_b64_decode_len(input.size())); size_t input_size = input.size(); - int output_size = - EVP_DecodeBase64(ret.data(), &out_len, out_len, reinterpret_cast(input.data()), input_size); - if (output_size <= 0) - return absl::nullopt; + size_t output_size = modp_b64_decode(reinterpret_cast(ret.data()), input.data(), input_size); + if (output_size == MODP_B64_ERROR) + return std::nullopt; ret.resize(output_size); return ret; diff --git a/yass/src/net/base64.hpp b/yass/src/net/base64.hpp index 0fbf3f7683..e4f87e6d48 100644 --- a/yass/src/net/base64.hpp +++ b/yass/src/net/base64.hpp @@ -11,19 +11,33 @@ #include #include +#include namespace net { // Encodes the input binary data in base64. -std::string Base64Encode(const uint8_t* data, size_t length); +std::string Base64Encode(absl::Span input); -// Encodes the input string in base64. -void Base64Encode(std::string_view input, std::string* output); +// Encodes the input binary data in base64 and appends it to the output. +void Base64EncodeAppend(absl::Span input, std::string* output); // Decodes the base64 input string. Returns true if successful and false // otherwise. The output string is only modified if successful. The decoding can // be done in-place. -bool Base64Decode(std::string_view input, std::string* output); +enum class Base64DecodePolicy { + // Input should match the output format of Base64Encode. i.e. + // - Input length should be divisible by 4 + // - Maximum of 2 padding characters + // - No non-base64 characters. + kStrict, + + // Matches https://infra.spec.whatwg.org/#forgiving-base64-decode. + // - Removes all ascii whitespace + // - Maximum of 2 padding characters + // - Allows input length not divisible by 4 if no padding chars are added. + kForgiving, +}; +bool Base64Decode(std::string_view input, std::string* output, Base64DecodePolicy policy = Base64DecodePolicy::kStrict); // Decodes the base64 input string. Returns `std::nullopt` if unsuccessful. std::optional> Base64Decode(std::string_view input); diff --git a/yass/src/net/http_parser.cpp b/yass/src/net/http_parser.cpp index 1aeca16562..de99a70c72 100644 --- a/yass/src/net/http_parser.cpp +++ b/yass/src/net/http_parser.cpp @@ -269,6 +269,9 @@ void HttpRequestParser::ProcessHeaders(const quiche::BalsaHeaders& headers) { if (key == "Proxy-Connection") { connection_ = std::string(value); } + if (key == "Proxy-Authorization") { + proxy_authorization_ = std::string(value); + } } } @@ -540,6 +543,9 @@ int HttpRequestParser::OnReadHttpRequestHeaderValue(http_parser* parser, const c if (self->http_field_ == "Content-Type") { self->content_type_ = std::string(buf, len); } + if (self->http_field_ == "Proxy-Authorization") { + self->proxy_authorization_ = std::string(buf, len); + } return 0; } diff --git a/yass/src/net/http_parser.hpp b/yass/src/net/http_parser.hpp index 47bdb11655..1f7394f15f 100644 --- a/yass/src/net/http_parser.hpp +++ b/yass/src/net/http_parser.hpp @@ -50,6 +50,7 @@ class HttpRequestParser : public quiche::BalsaVisitorInterface { uint64_t content_length() const { return headers_.content_length(); } const std::string& content_type() const { return content_type_; } const std::string& connection() const { return connection_; } + const std::string& proxy_authorization() const { return proxy_authorization_; } bool transfer_encoding_is_chunked() const { return headers_.transfer_encoding_is_chunked(); } void ReforgeHttpRequest(std::string* header, @@ -106,6 +107,8 @@ class HttpRequestParser : public quiche::BalsaVisitorInterface { std::string content_type_; /// copy of connection std::string connection_; + /// copy of proxy_authorization + std::string proxy_authorization_; bool first_byte_processed_ = false; bool headers_done_ = false; @@ -138,6 +141,7 @@ class HttpRequestParser { uint64_t content_length() const; const std::string& content_type() const { return content_type_; } std::string_view connection() const; + const std::string& proxy_authorization() const { return proxy_authorization_; } bool transfer_encoding_is_chunked() const; int status_code() const; @@ -170,6 +174,8 @@ class HttpRequestParser { bool http_is_connect_ = false; /// copy of content type std::string content_type_; + /// copy of proxy_authorization + std::string proxy_authorization_; }; class HttpResponseParser : public HttpRequestParser { diff --git a/yass/src/server/server_connection.cpp b/yass/src/server/server_connection.cpp index e9bdf1712a..0a520ad47f 100644 --- a/yass/src/server/server_connection.cpp +++ b/yass/src/server/server_connection.cpp @@ -54,11 +54,20 @@ static std::vector GenerateHeaders(std::vector TLSEXT_MAXLEN_host_name) { - LOG(WARNING) << "Connection (server) " << connection_id() << " too long domain name: " << http_host_; + LOG(INFO) << "Connection (server) " << connection_id() << " too long domain name: " << http_host_; + ec = asio::error::invalid_argument; + OnDisconnect(ec); + return; + } + + bool auth_required = !absl::GetFlag(FLAGS_username).empty() && !absl::GetFlag(FLAGS_password).empty(); + if (auth_required && !VerifyProxyAuthorizationIdentity(parser.proxy_authorization())) { + LOG(INFO) << "Connection (server) " << connection_id() << " Unexpected auth token."; ec = asio::error::invalid_argument; OnDisconnect(ec); return; diff --git a/yass/third_party/modp_b64/LICENSE b/yass/third_party/modp_b64/LICENSE new file mode 100644 index 0000000000..55af76f3eb --- /dev/null +++ b/yass/third_party/modp_b64/LICENSE @@ -0,0 +1,33 @@ + * MODP_B64 - High performance base64 encoder/decoder + * Version 1.3 -- 17-Mar-2006 + * http://modp.com/release/base64 + * + * Copyright (c) 2005, 2006 Nick Galbreath -- nickg [at] modp [dot] com + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * Neither the name of the modp.com nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/yass/third_party/modp_b64/modp_b64.cc b/yass/third_party/modp_b64/modp_b64.cc new file mode 100644 index 0000000000..9be58205f4 --- /dev/null +++ b/yass/third_party/modp_b64/modp_b64.cc @@ -0,0 +1,168 @@ +/* -*- mode: c; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */ +/* vi: set expandtab shiftwidth=4 tabstop=4: */ +/** + * \file + * 
+ * MODP_B64 - High performance base64 encoder/decoder
+ * Version 1.3 -- 17-Mar-2006
+ * http://modp.com/release/base64
+ *
+ * Copyright © 2005, 2006  Nick Galbreath -- nickg [at] modp [dot] com
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *   Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ *   Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ *   Neither the name of the modp.com nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * This is the standard "new" BSD license:
+ * http://www.opensource.org/licenses/bsd-license.php
+ *
+ */ + +/* public header */ +#include "modp_b64.h" + +#include "modp_b64_data.h" + +#define BADCHAR 0x01FFFFFF + +size_t modp_b64_encode_data(char* dest, const char* str, size_t len) +{ + size_t i = 0; + uint8_t* p = (uint8_t*) dest; + + /* unsigned here is important! */ + uint8_t t1, t2, t3; + + if (len > 2) { + for (; i < len - 2; i += 3) { + t1 = str[i]; t2 = str[i+1]; t3 = str[i+2]; + *p++ = e0[t1]; + *p++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *p++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; + *p++ = e2[t3]; + } + } + + switch (len - i) { + case 0: + break; + case 1: + t1 = str[i]; + *p++ = e0[t1]; + *p++ = e1[(t1 & 0x03) << 4]; + *p++ = CHARPAD; + *p++ = CHARPAD; + break; + default: /* case 2 */ + t1 = str[i]; t2 = str[i+1]; + *p++ = e0[t1]; + *p++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; + *p++ = e2[(t2 & 0x0F) << 2]; + *p++ = CHARPAD; + } + + return p - (uint8_t*)dest; +} + +size_t modp_b64_encode(char* dest, const char* str, size_t len) { + size_t output_size = modp_b64_encode_data(dest, str, len); + dest[output_size] = '\0'; + return output_size; +} + +size_t do_decode_padding(const char* src, size_t len, ModpDecodePolicy policy) { + if (policy == ModpDecodePolicy::kNoPaddingValidation) { + while (len > 0 && src[len - 1] == CHARPAD) { + len--; + } + } else { + const size_t remainder = len % 4; + if (policy == ModpDecodePolicy::kStrict && (remainder != 0 || len < 4)) + return MODP_B64_ERROR; + if (remainder == 0) { + if (src[len - 1] == CHARPAD) { + len--; + if (src[len - 1] == CHARPAD) { + len--; + } + } + } + } + return len % 4 == 1 ? MODP_B64_ERROR : len; +} + +size_t modp_b64_decode(char* dest, const char* src, size_t len, ModpDecodePolicy policy) +{ + if (len != 0) + len = do_decode_padding(src, len, policy); + + if (len == 0 || len == MODP_B64_ERROR) + return len; + + size_t i; + int leftover = len % 4; + size_t chunks = (leftover == 0) ? len / 4 - 1 : len /4; + + uint8_t* p = (uint8_t*)dest; + uint32_t x = 0; + const uint8_t* y = (uint8_t*)src; + for (i = 0; i < chunks; ++i, y += 4) { + x = d0[y[0]] | d1[y[1]] | d2[y[2]] | d3[y[3]]; + if (x >= BADCHAR) return MODP_B64_ERROR; + *p++ = ((uint8_t*)(&x))[0]; + *p++ = ((uint8_t*)(&x))[1]; + *p++ = ((uint8_t*)(&x))[2]; + } + + switch (leftover) { + case 0: + x = d0[y[0]] | d1[y[1]] | d2[y[2]] | d3[y[3]]; + + if (x >= BADCHAR) return MODP_B64_ERROR; + *p++ = ((uint8_t*)(&x))[0]; + *p++ = ((uint8_t*)(&x))[1]; + *p = ((uint8_t*)(&x))[2]; + return (chunks+1)*3; + case 1: /* with padding this is an impossible case */ + x = d0[y[0]]; + *p = *((uint8_t*)(&x)); // i.e. first char/byte in int + break; + case 2: // * case 2, 1 output byte */ + x = d0[y[0]] | d1[y[1]]; + *p = *((uint8_t*)(&x)); // i.e. first char + break; + default: /* case 3, 2 output bytes */ + x = d0[y[0]] | d1[y[1]] | d2[y[2]]; /* 0x3c */ + *p++ = ((uint8_t*)(&x))[0]; + *p = ((uint8_t*)(&x))[1]; + break; + } + + if (x >= BADCHAR) return MODP_B64_ERROR; + + return 3*chunks + (6*leftover)/8; +} diff --git a/yass/third_party/modp_b64/modp_b64.h b/yass/third_party/modp_b64/modp_b64.h new file mode 100644 index 0000000000..9d11f811d7 --- /dev/null +++ b/yass/third_party/modp_b64/modp_b64.h @@ -0,0 +1,159 @@ +/* -*- mode: c; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */ +/* vi: set expandtab shiftwidth=4 tabstop=4: */ + +/** + * \file + * 
+ * High performance base64 encoder / decoder
+ * Version 1.3 -- 17-Mar-2006
+ *
+ * Copyright © 2005, 2006, Nick Galbreath -- nickg [at] modp [dot] com
+ * All rights reserved.
+ *
+ * http://modp.com/release/base64
+ *
+ * Released under bsd license.  See modp_b64.c for details.
+ *
+ * + * The default implementation is the standard b64 encoding with padding. + * It's easy to change this to use "URL safe" characters and to remove + * padding. See the modp_b64.c source code for details. + * + */ + +#ifndef MODP_B64 +#define MODP_B64 + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Encode a raw binary string into base 64. + * src contains the bytes + * len contains the number of bytes in the src + * dest should be allocated by the caller to contain + * at least modp_b64_encode_len(len) bytes (see below) + * This will contain the (non-null terminated) b64 bytes. + * returns length of the destination string. + * + * Example + * + * \code + * char* src = ...; + * int srclen = ...; //the length of number of bytes in src + * char* dest = (char*) malloc(modp_b64_encode_len); + * int len = modp_b64_encode_data(dest, src, sourcelen); + * if (len == -1) { + * printf("Error\n"); + * } else { + * printf("b64 = %s\n", dest); + * } + * \endcode + * + */ +size_t modp_b64_encode_data(char* dest, const char* str, size_t len); + +/** + * Same as modp_b64_encode_data, but additionally sets a null terminator at the + * end of `dest` (i.e. at dest[output_size]). + * Like modp_b64_encode_data, returns the length of the destination string (i.e. + * not counting the null terminator). + * + * TODO(csharrison): Consider removing this once all callers migrate to + * modp_b64_encode_data. + */ +size_t modp_b64_encode(char* dest, const char* str, size_t len); + +/** + * Decode a base64 encoded string + * + * src should contain exactly len bytes of b64 characters. + * if src contains -any- non-base characters (such as white + * space, -1 is returned. + * + * dest should be allocated by the caller to contain at least + * len * 3 / 4 bytes. + * + * Returns the length (strlen) of the output, or -1 if unable to + * decode + * + * \code + * char* src = ...; + * int srclen = ...; // or if you don't know use strlen(src) + * char* dest = (char*) malloc(modp_b64_decode_len(srclen)); + * int len = modp_b64_decode(dest, src, sourcelen); + * if (len == -1) { error } + * \endcode + */ +enum class ModpDecodePolicy { + // src length must be divisible by 4, with a max of 2 pad chars. + kStrict, + + // Matches the infra spec: https://infra.spec.whatwg.org/#forgiving-base64 + // _except_ for ignoring whitespace (Step 1). + kForgiving, + + // src length % 4 must not equal 1, after stripping all pad chars. + // Accepts any number of pad chars. + kNoPaddingValidation, +}; +size_t modp_b64_decode( + char* dest, + const char* src, + size_t len, + ModpDecodePolicy policy = ModpDecodePolicy::kStrict); + +/** + * The maximum input that can be passed into modp_b64_encode{_data}. + * Lengths beyond this will overflow modp_b64_encode_len. + * + * This works because modp_b64_encode_len(A) computes: + * ceiling[max_len / 3] * 4 + 1 + * = ceiling[floor[(SIZE_MAX-1)/4]*3 / 3] * 4 + 1 + * = floor[(SIZE_MAX-1)/4] * 4 + 1 + * <= SIZE_MAX-1 + 1 + * = SIZE_MAX + * + * Note: technically modp_b64_encode_data can take one extra byte, but for + * simplicity the bound is shared between the two functions. + */ +#define MODP_B64_MAX_INPUT_LEN ((SIZE_MAX - 1) / 4 * 3) + +/** + * Given a source string of length len, this returns the amount of + * memory the destination string should have, for modp_b64_encode_data and + * modp_b64_encode, respectively. + * + * remember, this is integer math + * 3 bytes turn into 4 chars + * ceiling[len / 3] * 4 + * + * + * WARNING: These expressions will overflow if the A is above + * MODP_B64_MAX_INPUT_LEN. The caller must check this bound first. + */ +#define modp_b64_encode_data_len(A) ((A + 2) / 3 * 4) +#define modp_b64_encode_len(A) (modp_b64_encode_data_len(A) + 1) + +/** + * Given a base64 string of length len, + * this returns the amount of memory required for output string + * It maybe be more than the actual number of bytes written. + * NOTE: remember this is integer math + * this allocates a bit more memory than traditional versions of b64 + * decode 4 chars turn into 3 bytes + * floor[len * 3/4] + 2 + */ +#define modp_b64_decode_len(A) (A / 4 * 3 + 2) + +#define MODP_B64_ERROR ((size_t)-1) + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* MODP_B64 */ diff --git a/yass/third_party/modp_b64/modp_b64_data.h b/yass/third_party/modp_b64/modp_b64_data.h new file mode 100644 index 0000000000..2ecf5977b9 --- /dev/null +++ b/yass/third_party/modp_b64/modp_b64_data.h @@ -0,0 +1,481 @@ +#include + +#define CHAR62 '+' +#define CHAR63 '/' +#define CHARPAD '=' +static const char e0[256] = { + 'A', 'A', 'A', 'A', 'B', 'B', 'B', 'B', 'C', 'C', + 'C', 'C', 'D', 'D', 'D', 'D', 'E', 'E', 'E', 'E', + 'F', 'F', 'F', 'F', 'G', 'G', 'G', 'G', 'H', 'H', + 'H', 'H', 'I', 'I', 'I', 'I', 'J', 'J', 'J', 'J', + 'K', 'K', 'K', 'K', 'L', 'L', 'L', 'L', 'M', 'M', + 'M', 'M', 'N', 'N', 'N', 'N', 'O', 'O', 'O', 'O', + 'P', 'P', 'P', 'P', 'Q', 'Q', 'Q', 'Q', 'R', 'R', + 'R', 'R', 'S', 'S', 'S', 'S', 'T', 'T', 'T', 'T', + 'U', 'U', 'U', 'U', 'V', 'V', 'V', 'V', 'W', 'W', + 'W', 'W', 'X', 'X', 'X', 'X', 'Y', 'Y', 'Y', 'Y', + 'Z', 'Z', 'Z', 'Z', 'a', 'a', 'a', 'a', 'b', 'b', + 'b', 'b', 'c', 'c', 'c', 'c', 'd', 'd', 'd', 'd', + 'e', 'e', 'e', 'e', 'f', 'f', 'f', 'f', 'g', 'g', + 'g', 'g', 'h', 'h', 'h', 'h', 'i', 'i', 'i', 'i', + 'j', 'j', 'j', 'j', 'k', 'k', 'k', 'k', 'l', 'l', + 'l', 'l', 'm', 'm', 'm', 'm', 'n', 'n', 'n', 'n', + 'o', 'o', 'o', 'o', 'p', 'p', 'p', 'p', 'q', 'q', + 'q', 'q', 'r', 'r', 'r', 'r', 's', 's', 's', 's', + 't', 't', 't', 't', 'u', 'u', 'u', 'u', 'v', 'v', + 'v', 'v', 'w', 'w', 'w', 'w', 'x', 'x', 'x', 'x', + 'y', 'y', 'y', 'y', 'z', 'z', 'z', 'z', '0', '0', + '0', '0', '1', '1', '1', '1', '2', '2', '2', '2', + '3', '3', '3', '3', '4', '4', '4', '4', '5', '5', + '5', '5', '6', '6', '6', '6', '7', '7', '7', '7', + '8', '8', '8', '8', '9', '9', '9', '9', '+', '+', + '+', '+', '/', '/', '/', '/' +}; + +static const char e1[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', + 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', + 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', + 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', + 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', + 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', + 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', + 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', + 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', + '6', '7', '8', '9', '+', '/' +}; + +static const char e2[256] = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', + 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', + 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', + 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', + 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', + 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', + 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/', 'A', 'B', + 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', + 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', + 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + '+', '/', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', + 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', + 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', + 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', + '6', '7', '8', '9', '+', '/' +}; + + + +#ifdef WORDS_BIGENDIAN + + +/* SPECIAL DECODE TABLES FOR BIG ENDIAN (IBM/MOTOROLA/SUN) CPUS */ + +static const uint32_t d0[256] = { +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x00f80000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00fc0000, +0x00d00000, 0x00d40000, 0x00d80000, 0x00dc0000, 0x00e00000, 0x00e40000, +0x00e80000, 0x00ec0000, 0x00f00000, 0x00f40000, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, +0x00040000, 0x00080000, 0x000c0000, 0x00100000, 0x00140000, 0x00180000, +0x001c0000, 0x00200000, 0x00240000, 0x00280000, 0x002c0000, 0x00300000, +0x00340000, 0x00380000, 0x003c0000, 0x00400000, 0x00440000, 0x00480000, +0x004c0000, 0x00500000, 0x00540000, 0x00580000, 0x005c0000, 0x00600000, +0x00640000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x00680000, 0x006c0000, 0x00700000, 0x00740000, 0x00780000, +0x007c0000, 0x00800000, 0x00840000, 0x00880000, 0x008c0000, 0x00900000, +0x00940000, 0x00980000, 0x009c0000, 0x00a00000, 0x00a40000, 0x00a80000, +0x00ac0000, 0x00b00000, 0x00b40000, 0x00b80000, 0x00bc0000, 0x00c00000, +0x00c40000, 0x00c80000, 0x00cc0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff +}; + + +static const uint32_t d1[256] = { +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x0003e000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0003f000, +0x00034000, 0x00035000, 0x00036000, 0x00037000, 0x00038000, 0x00039000, +0x0003a000, 0x0003b000, 0x0003c000, 0x0003d000, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, +0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, +0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, +0x0000d000, 0x0000e000, 0x0000f000, 0x00010000, 0x00011000, 0x00012000, +0x00013000, 0x00014000, 0x00015000, 0x00016000, 0x00017000, 0x00018000, +0x00019000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x0001a000, 0x0001b000, 0x0001c000, 0x0001d000, 0x0001e000, +0x0001f000, 0x00020000, 0x00021000, 0x00022000, 0x00023000, 0x00024000, +0x00025000, 0x00026000, 0x00027000, 0x00028000, 0x00029000, 0x0002a000, +0x0002b000, 0x0002c000, 0x0002d000, 0x0002e000, 0x0002f000, 0x00030000, +0x00031000, 0x00032000, 0x00033000, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff +}; + + +static const uint32_t d2[256] = { +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x00000f80, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000fc0, +0x00000d00, 0x00000d40, 0x00000d80, 0x00000dc0, 0x00000e00, 0x00000e40, +0x00000e80, 0x00000ec0, 0x00000f00, 0x00000f40, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, +0x00000040, 0x00000080, 0x000000c0, 0x00000100, 0x00000140, 0x00000180, +0x000001c0, 0x00000200, 0x00000240, 0x00000280, 0x000002c0, 0x00000300, +0x00000340, 0x00000380, 0x000003c0, 0x00000400, 0x00000440, 0x00000480, +0x000004c0, 0x00000500, 0x00000540, 0x00000580, 0x000005c0, 0x00000600, +0x00000640, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x00000680, 0x000006c0, 0x00000700, 0x00000740, 0x00000780, +0x000007c0, 0x00000800, 0x00000840, 0x00000880, 0x000008c0, 0x00000900, +0x00000940, 0x00000980, 0x000009c0, 0x00000a00, 0x00000a40, 0x00000a80, +0x00000ac0, 0x00000b00, 0x00000b40, 0x00000b80, 0x00000bc0, 0x00000c00, +0x00000c40, 0x00000c80, 0x00000cc0, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff +}; + + +static const uint32_t d3[256] = { +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x0000003e, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000003f, +0x00000034, 0x00000035, 0x00000036, 0x00000037, 0x00000038, 0x00000039, +0x0000003a, 0x0000003b, 0x0000003c, 0x0000003d, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, +0x00000001, 0x00000002, 0x00000003, 0x00000004, 0x00000005, 0x00000006, +0x00000007, 0x00000008, 0x00000009, 0x0000000a, 0x0000000b, 0x0000000c, +0x0000000d, 0x0000000e, 0x0000000f, 0x00000010, 0x00000011, 0x00000012, +0x00000013, 0x00000014, 0x00000015, 0x00000016, 0x00000017, 0x00000018, +0x00000019, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x0000001a, 0x0000001b, 0x0000001c, 0x0000001d, 0x0000001e, +0x0000001f, 0x00000020, 0x00000021, 0x00000022, 0x00000023, 0x00000024, +0x00000025, 0x00000026, 0x00000027, 0x00000028, 0x00000029, 0x0000002a, +0x0000002b, 0x0000002c, 0x0000002d, 0x0000002e, 0x0000002f, 0x00000030, +0x00000031, 0x00000032, 0x00000033, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff +}; + + +#else + + +/* SPECIAL DECODE TABLES FOR LITTLE ENDIAN (INTEL) CPUS */ + +static const uint32_t d0[256] = { +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x000000f8, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x000000fc, +0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4, +0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, +0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018, +0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030, +0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048, +0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060, +0x00000064, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078, +0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090, +0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8, +0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0, +0x000000c4, 0x000000c8, 0x000000cc, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff +}; + + +static const uint32_t d1[256] = { +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x0000e003, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x0000f003, +0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003, +0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, +0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000, +0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000, +0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001, +0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001, +0x00009001, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001, +0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002, +0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002, +0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003, +0x00001003, 0x00002003, 0x00003003, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff +}; + + +static const uint32_t d2[256] = { +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x00800f00, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00c00f00, +0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00, +0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, +0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100, +0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300, +0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400, +0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600, +0x00400600, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700, +0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900, +0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00, +0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00, +0x00400c00, 0x00800c00, 0x00c00c00, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff +}; + + +static const uint32_t d3[256] = { +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x003e0000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x003f0000, +0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000, +0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x00000000, +0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000, +0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000, +0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000, +0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000, +0x00190000, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000, +0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000, +0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000, +0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000, +0x00310000, 0x00320000, 0x00330000, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff, +0x01ffffff, 0x01ffffff, 0x01ffffff, 0x01ffffff +}; + + +#endif diff --git a/yass/yass.spec.in b/yass/yass.spec.in index f7bb2fcea0..c885d4ad6f 100644 --- a/yass/yass.spec.in +++ b/yass/yass.spec.in @@ -259,6 +259,8 @@ for embedded devices and low end boxes. %systemd_postun_with_restart yass-redir.service %changelog +* Sun May 26 2024 Chilledheart - 1.10.3-1 + - net: support https protocol via caddy. * Thu May 23 2024 Chilledheart - 1.10.2-1 - net: reduce memory footprint peak and cpu peak. * Sun May 19 2024 Chilledheart - 1.10.1-1 diff --git a/yt-dlp/README.md b/yt-dlp/README.md index 887cfde231..0636d2f6e7 100644 --- a/yt-dlp/README.md +++ b/yt-dlp/README.md @@ -1835,6 +1835,9 @@ The following extractors use this feature: #### nflplusreplay * `type`: Type(s) of game replays to extract. Valid types are: `full_game`, `full_game_spanish`, `condensed_game` and `all_22`. You can use `all` to extract all available replay types, which is the default +#### jiocinema +* `refresh_token`: The `refreshToken` UUID from browser local storage can be passed to extend the life of your login session when logging in with `token` as username and the `accessToken` from browser local storage as password + #### jiosaavn * `bitrate`: Audio bitrates to request. One or more of `16`, `32`, `64`, `128`, `320`. Default is `128,320` diff --git a/yt-dlp/yt_dlp/extractor/_extractors.py b/yt-dlp/yt_dlp/extractor/_extractors.py index 6f0656e0c3..fc18ead3a9 100644 --- a/yt-dlp/yt_dlp/extractor/_extractors.py +++ b/yt-dlp/yt_dlp/extractor/_extractors.py @@ -453,6 +453,10 @@ from .damtomo import ( DamtomoRecordIE, DamtomoVideoIE, ) +from .dangalplay import ( + DangalPlayIE, + DangalPlaySeasonIE, +) from .daum import ( DaumIE, DaumClipIE, @@ -872,6 +876,10 @@ from .japandiet import ( SangiinIE, ) from .jeuxvideo import JeuxVideoIE +from .jiocinema import ( + JioCinemaIE, + JioCinemaSeriesIE, +) from .jiosaavn import ( JioSaavnSongIE, JioSaavnAlbumIE, @@ -2282,10 +2290,6 @@ from .voicy import ( VoicyChannelIE, ) from .volejtv import VolejTVIE -from .voot import ( - VootIE, - VootSeriesIE, -) from .voxmedia import ( VoxMediaVolumeIE, VoxMediaIE, @@ -2382,6 +2386,7 @@ from .xhamster import ( XHamsterEmbedIE, XHamsterUserIE, ) +from .xiaohongshu import XiaoHongShuIE from .ximalaya import ( XimalayaIE, XimalayaAlbumIE diff --git a/yt-dlp/yt_dlp/extractor/dangalplay.py b/yt-dlp/yt_dlp/extractor/dangalplay.py new file mode 100644 index 0000000000..50e4136b57 --- /dev/null +++ b/yt-dlp/yt_dlp/extractor/dangalplay.py @@ -0,0 +1,197 @@ +import hashlib +import json +import re +import time + +from .common import InfoExtractor +from ..networking.exceptions import HTTPError +from ..utils import ExtractorError, int_or_none, join_nonempty, url_or_none +from ..utils.traversal import traverse_obj + + +class DangalPlayBaseIE(InfoExtractor): + _NETRC_MACHINE = 'dangalplay' + _OTV_USER_ID = None + _LOGIN_HINT = 'Pass credentials as -u "token" -p "USER_ID" where USER_ID is the `otv_user_id` in browser local storage' + _API_BASE = 'https://ottapi.dangalplay.com' + _AUTH_TOKEN = 'jqeGWxRKK7FK5zEk3xCM' # from https://www.dangalplay.com/main.48ad19e24eb46acccef3.js + _SECRET_KEY = 'f53d31a4377e4ef31fa0' # same as above + + def _perform_login(self, username, password): + if self._OTV_USER_ID: + return + if username != 'token' or not re.fullmatch(r'[\da-f]{32}', password): + raise ExtractorError(self._LOGIN_HINT, expected=True) + self._OTV_USER_ID = password + + def _real_initialize(self): + if not self._OTV_USER_ID: + self.raise_login_required(f'Login required. {self._LOGIN_HINT}', method=None) + + def _extract_episode_info(self, metadata, episode_slug, series_slug): + return { + 'display_id': episode_slug, + 'episode_number': int_or_none(self._search_regex( + r'ep-(?:number-)?(\d+)', episode_slug, 'episode number', default=None)), + 'season_number': int_or_none(self._search_regex( + r'season-(\d+)', series_slug, 'season number', default='1')), + 'series': series_slug, + **traverse_obj(metadata, { + 'id': ('content_id', {str}), + 'title': ('display_title', {str}), + 'episode': ('title', {str}), + 'series': ('show_name', {str}, {lambda x: x or None}), + 'series_id': ('catalog_id', {str}), + 'duration': ('duration', {int_or_none}), + 'release_timestamp': ('release_date_uts', {int_or_none}), + }), + } + + def _call_api(self, path, display_id, note='Downloading JSON metadata', fatal=True, query={}): + return self._download_json( + f'{self._API_BASE}/{path}', display_id, note, fatal=fatal, + headers={'Accept': 'application/json'}, query={ + 'auth_token': self._AUTH_TOKEN, + 'region': 'IN', + **query, + }) + + +class DangalPlayIE(DangalPlayBaseIE): + IE_NAME = 'dangalplay' + _VALID_URL = r'https?://(?:www\.)?dangalplay.com/shows/(?P[^/?#]+)/(?P(?!episodes)[^/?#]+)/?(?:$|[?#])' + _TESTS = [{ + 'url': 'https://www.dangalplay.com/shows/kitani-mohabbat-hai-season-2/kitani-mohabbat-hai-season-2-ep-number-01', + 'info_dict': { + 'id': '647c61dc1e7171310dcd49b4', + 'ext': 'mp4', + 'release_timestamp': 1262304000, + 'episode_number': 1, + 'episode': 'EP 1 | KITANI MOHABBAT HAI SEASON 2', + 'series': 'kitani-mohabbat-hai-season-2', + 'season_number': 2, + 'title': 'EP 1 | KITANI MOHABBAT HAI SEASON 2', + 'release_date': '20100101', + 'duration': 2325, + 'season': 'Season 2', + 'display_id': 'kitani-mohabbat-hai-season-2-ep-number-01', + 'series_id': '645c9ea41e717158ca574966', + }, + }, { + 'url': 'https://www.dangalplay.com/shows/milke-bhi-hum-na-mile/milke-bhi-hum-na-mile-ep-number-01', + 'info_dict': { + 'id': '65d31d9ba73b9c3abd14a7f3', + 'ext': 'mp4', + 'episode': 'EP 1 | MILKE BHI HUM NA MILE', + 'release_timestamp': 1708367411, + 'episode_number': 1, + 'season': 'Season 1', + 'title': 'EP 1 | MILKE BHI HUM NA MILE', + 'duration': 156048, + 'release_date': '20240219', + 'season_number': 1, + 'series': 'MILKE BHI HUM NA MILE', + 'series_id': '645c9ea41e717158ca574966', + 'display_id': 'milke-bhi-hum-na-mile-ep-number-01', + }, + }] + + def _generate_api_data(self, data): + catalog_id = data['catalog_id'] + content_id = data['content_id'] + timestamp = str(int(time.time())) + unhashed = ''.join((catalog_id, content_id, self._OTV_USER_ID, timestamp, self._SECRET_KEY)) + + return json.dumps({ + 'catalog_id': catalog_id, + 'content_id': content_id, + 'category': '', + 'region': 'IN', + 'auth_token': self._AUTH_TOKEN, + 'id': self._OTV_USER_ID, + 'md5': hashlib.md5(unhashed.encode()).hexdigest(), + 'ts': timestamp, + }, separators=(',', ':')).encode() + + def _real_extract(self, url): + series_slug, episode_slug = self._match_valid_url(url).group('series', 'id') + metadata = self._call_api( + f'catalogs/shows/{series_slug}/episodes/{episode_slug}.gzip', + episode_slug, query={'item_language': ''})['data'] + + try: + details = self._download_json( + f'{self._API_BASE}/v2/users/get_all_details.gzip', episode_slug, + 'Downloading playback details JSON', headers={ + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }, data=self._generate_api_data(metadata))['data'] + except ExtractorError as e: + if isinstance(e.cause, HTTPError) and e.cause.status == 422: + error_info = traverse_obj(e.cause.response.read().decode(), ({json.loads}, 'error', {dict})) or {} + if error_info.get('code') == '1016': + self.raise_login_required( + f'Your token has expired or is invalid. {self._LOGIN_HINT}', method=None) + elif msg := error_info.get('message'): + raise ExtractorError(msg) + raise + + m3u8_url = traverse_obj(details, ( + ('adaptive_url', ('adaptive_urls', 'hd', 'hls', ..., 'playback_url')), {url_or_none}, any)) + formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, episode_slug, 'mp4') + + return { + 'formats': formats, + 'subtitles': subtitles, + **self._extract_episode_info(metadata, episode_slug, series_slug), + } + + +class DangalPlaySeasonIE(DangalPlayBaseIE): + IE_NAME = 'dangalplay:season' + _VALID_URL = r'https?://(?:www\.)?dangalplay.com/shows/(?P[^/?#]+)(?:/(?Pep-[^/?#]+)/episodes)?/?(?:$|[?#])' + _TESTS = [{ + 'url': 'https://www.dangalplay.com/shows/kitani-mohabbat-hai-season-1', + 'playlist_mincount': 170, + 'info_dict': { + 'id': 'kitani-mohabbat-hai-season-1', + }, + }, { + 'url': 'https://www.dangalplay.com/shows/kitani-mohabbat-hai-season-1/ep-01-30-1/episodes', + 'playlist_count': 30, + 'info_dict': { + 'id': 'kitani-mohabbat-hai-season-1-ep-01-30-1', + }, + }, { + # 1 season only, series page is season page + 'url': 'https://www.dangalplay.com/shows/milke-bhi-hum-na-mile', + 'playlist_mincount': 15, + 'info_dict': { + 'id': 'milke-bhi-hum-na-mile', + }, + }] + + def _entries(self, subcategories, series_slug): + for subcategory in subcategories: + data = self._call_api( + f'catalogs/shows/items/{series_slug}/subcategories/{subcategory}/episodes.gzip', + series_slug, f'Downloading episodes JSON for {subcategory}', fatal=False, query={ + 'order_by': 'asc', + 'status': 'published', + }) + for ep in traverse_obj(data, ('data', 'items', lambda _, v: v['friendly_id'])): + episode_slug = ep['friendly_id'] + yield self.url_result( + f'https://www.dangalplay.com/shows/{series_slug}/{episode_slug}', + DangalPlayIE, **self._extract_episode_info(ep, episode_slug, series_slug)) + + def _real_extract(self, url): + series_slug, subcategory = self._match_valid_url(url).group('id', 'sub') + subcategories = [subcategory] if subcategory else traverse_obj( + self._call_api( + f'catalogs/shows/items/{series_slug}.gzip', series_slug, + 'Downloading season info JSON', query={'item_language': ''}), + ('data', 'subcategories', ..., 'friendly_id', {str})) + + return self.playlist_result( + self._entries(subcategories, series_slug), join_nonempty(series_slug, subcategory)) diff --git a/yt-dlp/yt_dlp/extractor/jiocinema.py b/yt-dlp/yt_dlp/extractor/jiocinema.py new file mode 100644 index 0000000000..e7186d75c5 --- /dev/null +++ b/yt-dlp/yt_dlp/extractor/jiocinema.py @@ -0,0 +1,403 @@ +import base64 +import itertools +import json +import random +import re +import string +import time + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + float_or_none, + int_or_none, + jwt_decode_hs256, + parse_age_limit, + try_call, + url_or_none, +) +from ..utils.traversal import traverse_obj + + +class JioCinemaBaseIE(InfoExtractor): + _NETRC_MACHINE = 'jiocinema' + _GEO_BYPASS = False + _ACCESS_TOKEN = None + _REFRESH_TOKEN = None + _GUEST_TOKEN = None + _USER_ID = None + _DEVICE_ID = None + _API_HEADERS = {'Origin': 'https://www.jiocinema.com', 'Referer': 'https://www.jiocinema.com/'} + _APP_NAME = {'appName': 'RJIL_JioCinema'} + _APP_VERSION = {'appVersion': '5.0.0'} + _API_SIGNATURES = 'o668nxgzwff' + _METADATA_API_BASE = 'https://content-jiovoot.voot.com/psapi' + _ACCESS_HINT = 'the `accessToken` from your browser local storage' + _LOGIN_HINT = ( + 'Log in with "-u phone -p " to authenticate with OTP, ' + f'or use "-u token -p " to log in with {_ACCESS_HINT}. ' + 'If you have previously logged in with yt-dlp and your session ' + 'has been cached, you can use "-u device -p "') + + def _cache_token(self, token_type): + assert token_type in ('access', 'refresh', 'all') + if token_type in ('access', 'all'): + self.cache.store( + JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-access', JioCinemaBaseIE._ACCESS_TOKEN) + if token_type in ('refresh', 'all'): + self.cache.store( + JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-refresh', JioCinemaBaseIE._REFRESH_TOKEN) + + def _call_api(self, url, video_id, note='Downloading API JSON', headers={}, data={}): + return self._download_json( + url, video_id, note, data=json.dumps(data, separators=(',', ':')).encode(), headers={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + **self._API_HEADERS, + **headers, + }, expected_status=(400, 403, 474)) + + def _call_auth_api(self, service, endpoint, note, headers={}, data={}): + return self._call_api( + f'https://auth-jiocinema.voot.com/{service}service/apis/v4/{endpoint}', + None, note=note, headers=headers, data=data) + + def _refresh_token(self): + if not JioCinemaBaseIE._REFRESH_TOKEN or not JioCinemaBaseIE._DEVICE_ID: + raise ExtractorError('User token has expired', expected=True) + response = self._call_auth_api( + 'token', 'refreshtoken', 'Refreshing token', + headers={'accesstoken': self._ACCESS_TOKEN}, data={ + **self._APP_NAME, + 'deviceId': self._DEVICE_ID, + 'refreshToken': self._REFRESH_TOKEN, + **self._APP_VERSION, + }) + refresh_token = response.get('refreshTokenId') + if refresh_token and refresh_token != JioCinemaBaseIE._REFRESH_TOKEN: + JioCinemaBaseIE._REFRESH_TOKEN = refresh_token + self._cache_token('refresh') + JioCinemaBaseIE._ACCESS_TOKEN = response['authToken'] + self._cache_token('access') + + def _fetch_guest_token(self): + JioCinemaBaseIE._DEVICE_ID = ''.join(random.choices(string.digits, k=10)) + guest_token = self._call_auth_api( + 'token', 'guest', 'Downloading guest token', data={ + **self._APP_NAME, + 'deviceType': 'phone', + 'os': 'ios', + 'deviceId': self._DEVICE_ID, + 'freshLaunch': False, + 'adId': self._DEVICE_ID, + **self._APP_VERSION, + }) + self._GUEST_TOKEN = guest_token['authToken'] + self._USER_ID = guest_token['userId'] + + def _call_login_api(self, endpoint, guest_token, data, note): + return self._call_auth_api( + 'user', f'loginotp/{endpoint}', note, headers={ + **self.geo_verification_headers(), + 'accesstoken': self._GUEST_TOKEN, + **self._APP_NAME, + **traverse_obj(guest_token, 'data', { + 'deviceType': ('deviceType', {str}), + 'os': ('os', {str}), + })}, data=data) + + def _is_token_expired(self, token): + return (try_call(lambda: jwt_decode_hs256(token)['exp']) or 0) <= int(time.time() - 180) + + def _perform_login(self, username, password): + if self._ACCESS_TOKEN and not self._is_token_expired(self._ACCESS_TOKEN): + return + + UUID_RE = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}' + + if username.lower() == 'token': + if try_call(lambda: jwt_decode_hs256(password)): + JioCinemaBaseIE._ACCESS_TOKEN = password + refresh_hint = 'the `refreshToken` UUID from your browser local storage' + refresh_token = self._configuration_arg('refresh_token', [''], ie_key=JioCinemaIE)[0] + if not refresh_token: + self.to_screen( + 'To extend the life of your login session, in addition to your access token, ' + 'you can pass --extractor-args "jiocinema:refresh_token=REFRESH_TOKEN" ' + f'where REFRESH_TOKEN is {refresh_hint}') + elif re.fullmatch(UUID_RE, refresh_token): + JioCinemaBaseIE._REFRESH_TOKEN = refresh_token + else: + self.report_warning(f'Invalid refresh_token value. Use {refresh_hint}') + else: + raise ExtractorError( + f'The password given could not be decoded as a token; use {self._ACCESS_HINT}', expected=True) + + elif username.lower() == 'device' and re.fullmatch(rf'(?:{UUID_RE}|\d+)', password): + JioCinemaBaseIE._REFRESH_TOKEN = self.cache.load(JioCinemaBaseIE._NETRC_MACHINE, f'{password}-refresh') + JioCinemaBaseIE._ACCESS_TOKEN = self.cache.load(JioCinemaBaseIE._NETRC_MACHINE, f'{password}-access') + if not JioCinemaBaseIE._REFRESH_TOKEN or not JioCinemaBaseIE._ACCESS_TOKEN: + raise ExtractorError(f'Failed to load cached tokens for device ID "{password}"', expected=True) + + elif username.lower() == 'phone' and re.fullmatch(r'\+?\d+', password): + self._fetch_guest_token() + guest_token = jwt_decode_hs256(self._GUEST_TOKEN) + initial_data = { + 'number': base64.b64encode(password.encode()).decode(), + **self._APP_VERSION, + } + response = self._call_login_api('send', guest_token, initial_data, 'Requesting OTP') + if not traverse_obj(response, ('OTPInfo', {dict})): + raise ExtractorError('There was a problem with the phone number login attempt') + + is_iphone = guest_token.get('os') == 'ios' + response = self._call_login_api('verify', guest_token, { + 'deviceInfo': { + 'consumptionDeviceName': 'iPhone' if is_iphone else 'Android', + 'info': { + 'platform': {'name': 'iPhone OS' if is_iphone else 'Android'}, + 'androidId': self._DEVICE_ID, + 'type': 'iOS' if is_iphone else 'Android' + } + }, + **initial_data, + 'otp': self._get_tfa_info('the one-time password sent to your phone') + }, 'Submitting OTP') + if traverse_obj(response, 'code') == 1043: + raise ExtractorError('Wrong OTP', expected=True) + JioCinemaBaseIE._REFRESH_TOKEN = response['refreshToken'] + JioCinemaBaseIE._ACCESS_TOKEN = response['authToken'] + + else: + raise ExtractorError(self._LOGIN_HINT, expected=True) + + user_token = jwt_decode_hs256(JioCinemaBaseIE._ACCESS_TOKEN)['data'] + JioCinemaBaseIE._USER_ID = user_token['userId'] + JioCinemaBaseIE._DEVICE_ID = user_token['deviceId'] + if JioCinemaBaseIE._REFRESH_TOKEN and username != 'device': + self._cache_token('all') + if self.get_param('cachedir') is not False: + self.to_screen( + f'NOTE: For subsequent logins you can use "-u device -p {JioCinemaBaseIE._DEVICE_ID}"') + elif not JioCinemaBaseIE._REFRESH_TOKEN: + JioCinemaBaseIE._REFRESH_TOKEN = self.cache.load( + JioCinemaBaseIE._NETRC_MACHINE, f'{JioCinemaBaseIE._DEVICE_ID}-refresh') + if JioCinemaBaseIE._REFRESH_TOKEN: + self._cache_token('access') + self.to_screen(f'Logging in as device ID "{JioCinemaBaseIE._DEVICE_ID}"') + if self._is_token_expired(JioCinemaBaseIE._ACCESS_TOKEN): + self._refresh_token() + + +class JioCinemaIE(JioCinemaBaseIE): + IE_NAME = 'jiocinema' + _VALID_URL = r'https?://(?:www\.)?jiocinema\.com/?(?:movies?/[^/?#]+/|tv-shows/(?:[^/?#]+/){3})(?P\d{3,})' + _TESTS = [{ + 'url': 'https://www.jiocinema.com/tv-shows/agnisakshi-ek-samjhauta/1/pradeep-to-stop-the-wedding/3759931', + 'info_dict': { + 'id': '3759931', + 'ext': 'mp4', + 'title': 'Pradeep to stop the wedding?', + 'description': 'md5:75f72d1d1a66976633345a3de6d672b1', + 'episode': 'Pradeep to stop the wedding?', + 'episode_number': 89, + 'season': 'Agnisakshi…Ek Samjhauta-S1', + 'season_number': 1, + 'series': 'Agnisakshi Ek Samjhauta', + 'duration': 1238.0, + 'thumbnail': r're:https?://.+\.jpg', + 'age_limit': 13, + 'season_id': '3698031', + 'upload_date': '20230606', + 'timestamp': 1686009600, + 'release_date': '20230607', + 'genres': ['Drama'], + }, + 'params': {'skip_download': 'm3u8'}, + }, { + 'url': 'https://www.jiocinema.com/movies/bhediya/3754021/watch', + 'info_dict': { + 'id': '3754021', + 'ext': 'mp4', + 'title': 'Bhediya', + 'description': 'md5:a6bf2900371ac2fc3f1447401a9f7bb0', + 'episode': 'Bhediya', + 'duration': 8500.0, + 'thumbnail': r're:https?://.+\.jpg', + 'age_limit': 13, + 'upload_date': '20230525', + 'timestamp': 1685026200, + 'release_date': '20230524', + 'genres': ['Comedy'], + }, + 'params': {'skip_download': 'm3u8'}, + }] + + def _extract_formats_and_subtitles(self, playback, video_id): + m3u8_url = traverse_obj(playback, ( + 'data', 'playbackUrls', lambda _, v: v['streamtype'] == 'hls', 'url', {url_or_none}, any)) + if not m3u8_url: # DRM-only content only serves dash urls + self.report_drm(video_id) + formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, m3u8_id='hls') + self._remove_duplicate_formats(formats) + + return { + # '/_definst_/smil:vod/' m3u8 manifests claim to have 720p+ formats but max out at 480p + 'formats': traverse_obj(formats, ( + lambda _, v: '/_definst_/smil:vod/' not in v['url'] or v['height'] <= 480)), + 'subtitles': subtitles, + } + + def _real_extract(self, url): + video_id = self._match_id(url) + if not self._ACCESS_TOKEN and self._is_token_expired(self._GUEST_TOKEN): + self._fetch_guest_token() + elif self._ACCESS_TOKEN and self._is_token_expired(self._ACCESS_TOKEN): + self._refresh_token() + + playback = self._call_api( + f'https://apis-jiovoot.voot.com/playbackjv/v3/{video_id}', video_id, + 'Downloading playback JSON', headers={ + **self.geo_verification_headers(), + 'accesstoken': self._ACCESS_TOKEN or self._GUEST_TOKEN, + **self._APP_NAME, + 'deviceid': self._DEVICE_ID, + 'uniqueid': self._USER_ID, + 'x-apisignatures': self._API_SIGNATURES, + 'x-platform': 'androidweb', + 'x-platform-token': 'web', + }, data={ + '4k': False, + 'ageGroup': '18+', + 'appVersion': '3.4.0', + 'bitrateProfile': 'xhdpi', + 'capability': { + 'drmCapability': { + 'aesSupport': 'yes', + 'fairPlayDrmSupport': 'none', + 'playreadyDrmSupport': 'none', + 'widevineDRMSupport': 'none' + }, + 'frameRateCapability': [{ + 'frameRateSupport': '30fps', + 'videoQuality': '1440p' + }] + }, + 'continueWatchingRequired': False, + 'dolby': False, + 'downloadRequest': False, + 'hevc': False, + 'kidsSafe': False, + 'manufacturer': 'Windows', + 'model': 'Windows', + 'multiAudioRequired': True, + 'osVersion': '10', + 'parentalPinValid': True, + 'x-apisignatures': self._API_SIGNATURES + }) + + status_code = traverse_obj(playback, ('code', {int})) + if status_code == 474: + self.raise_geo_restricted(countries=['IN']) + elif status_code == 1008: + error_msg = 'This content is only available for premium users' + if self._ACCESS_TOKEN: + raise ExtractorError(error_msg, expected=True) + self.raise_login_required(f'{error_msg}. {self._LOGIN_HINT}', method=None) + elif status_code == 400: + raise ExtractorError('The requested content is not available', expected=True) + elif status_code is not None and status_code != 200: + raise ExtractorError( + f'JioCinema says: {traverse_obj(playback, ("message", {str})) or status_code}') + + metadata = self._download_json( + f'{self._METADATA_API_BASE}/voot/v1/voot-web/content/query/asset-details', + video_id, fatal=False, query={ + 'ids': f'include:{video_id}', + 'responseType': 'common', + 'devicePlatformType': 'desktop', + }) + + return { + 'id': video_id, + 'http_headers': self._API_HEADERS, + **self._extract_formats_and_subtitles(playback, video_id), + **traverse_obj(playback, ('data', { + # fallback metadata + 'title': ('name', {str}), + 'description': ('fullSynopsis', {str}), + 'series': ('show', 'name', {str}, {lambda x: x or None}), + 'season': ('tournamentName', {str}, {lambda x: x if x != 'Season 0' else None}), + 'season_number': ('episode', 'season', {int_or_none}, {lambda x: x or None}), + 'episode': ('fullTitle', {str}), + 'episode_number': ('episode', 'episodeNo', {int_or_none}, {lambda x: x or None}), + 'age_limit': ('ageNemonic', {parse_age_limit}), + 'duration': ('totalDuration', {float_or_none}), + 'thumbnail': ('images', {url_or_none}), + })), + **traverse_obj(metadata, ('result', 0, { + 'title': ('fullTitle', {str}), + 'description': ('fullSynopsis', {str}), + 'series': ('showName', {str}, {lambda x: x or None}), + 'season': ('seasonName', {str}, {lambda x: x or None}), + 'season_number': ('season', {int_or_none}), + 'season_id': ('seasonId', {str}, {lambda x: x or None}), + 'episode': ('fullTitle', {str}), + 'episode_number': ('episode', {int_or_none}), + 'timestamp': ('uploadTime', {int_or_none}), + 'release_date': ('telecastDate', {str}), + 'age_limit': ('ageNemonic', {parse_age_limit}), + 'duration': ('duration', {float_or_none}), + 'genres': ('genres', ..., {str}), + 'thumbnail': ('seo', 'ogImage', {url_or_none}), + })), + } + + +class JioCinemaSeriesIE(JioCinemaBaseIE): + IE_NAME = 'jiocinema:series' + _VALID_URL = r'https?://(?:www\.)?jiocinema\.com/tv-shows/(?P[\w-]+)/(?P\d{3,})' + _TESTS = [{ + 'url': 'https://www.jiocinema.com/tv-shows/naagin/3499917', + 'info_dict': { + 'id': '3499917', + 'title': 'naagin', + }, + 'playlist_mincount': 120, + }] + + def _entries(self, series_id): + seasons = self._download_json( + f'{self._METADATA_API_BASE}/voot/v1/voot-web/content/generic/season-by-show', series_id, + 'Downloading series metadata JSON', query={ + 'sort': 'season:asc', + 'id': series_id, + 'responseType': 'common', + }) + + for season_num, season in enumerate(traverse_obj(seasons, ('result', lambda _, v: v['id'])), 1): + season_id = season['id'] + label = season.get('season') or season_num + for page_num in itertools.count(1): + episodes = traverse_obj(self._download_json( + f'{self._METADATA_API_BASE}/voot/v1/voot-web/content/generic/series-wise-episode', + season_id, f'Downloading season {label} page {page_num} JSON', query={ + 'sort': 'episode:asc', + 'id': season_id, + 'responseType': 'common', + 'page': page_num, + }), ('result', lambda _, v: v['id'] and url_or_none(v['slug']))) + if not episodes: + break + for episode in episodes: + yield self.url_result( + episode['slug'], JioCinemaIE, **traverse_obj(episode, { + 'video_id': 'id', + 'video_title': ('fullTitle', {str}), + 'season_number': ('season', {int_or_none}), + 'episode_number': ('episode', {int_or_none}), + })) + + def _real_extract(self, url): + slug, series_id = self._match_valid_url(url).group('slug', 'id') + return self.playlist_result(self._entries(series_id), series_id, slug) diff --git a/yt-dlp/yt_dlp/extractor/piapro.py b/yt-dlp/yt_dlp/extractor/piapro.py index 3ae985da2b..87d912d568 100644 --- a/yt-dlp/yt_dlp/extractor/piapro.py +++ b/yt-dlp/yt_dlp/extractor/piapro.py @@ -2,6 +2,8 @@ from .common import InfoExtractor from ..compat import compat_urlparse from ..utils import ( ExtractorError, + clean_html, + get_element_by_class, parse_duration, parse_filesize, str_to_int, @@ -88,34 +90,22 @@ class PiaproIE(InfoExtractor): if category_id not in ('1', '2', '21', '22', '23', '24', '25'): raise ExtractorError('The URL does not contain audio.', expected=True) - str_duration, str_filesize = self._search_regex( - r'サイズ：(.+?)/\(([0-9,]+?[KMG]?B)）', webpage, 'duration and size', - group=(1, 2), default=(None, None)) - str_viewcount = self._search_regex(r'閲覧数：([0-9,]+)\s+', webpage, 'view count', fatal=False) - - uploader_id, uploader = self._search_regex( - r'([^<]+)さん<', webpage, 'uploader', - group=(1, 2), default=(None, None)) - content_id = self._search_regex(r'contentId\:\'(.+)\'', webpage, 'content ID') - create_date = self._search_regex(r'createDate\:\'(.+)\'', webpage, 'timestamp') - - player_webpage = self._download_webpage( - f'https://piapro.jp/html5_player_popup/?id={content_id}&cdate={create_date}', - video_id, note='Downloading player webpage') + def extract_info(name, description): + return self._search_regex(rf'{name}[：:]\s*([\d\s,:/]+)\s*

', webpage, description, default=None) return { 'id': video_id, - 'title': self._html_search_regex(r'(.+?)', webpage, 'title', fatal=False), - 'description': self._html_search_regex(r'(?s)(.+?)

\s*[^/?#&]+)' - _GEO_COUNTRIES = ['DE'] +class Tele5IE(DiscoveryPlusBaseIE): + _VALID_URL = r'https?://(?:www\.)?tele5\.de/(?P[\w-]+)/(?P[\w-]+)(?:/(?P[\w-]+))?' _TESTS = [{ - 'url': 'https://www.tele5.de/mediathek/filme-online/videos?vid=1549416', + # slug_a and slug_b + 'url': 'https://tele5.de/mediathek/stargate-atlantis/quarantane', 'info_dict': { - 'id': '1549416', + 'id': '6852024', 'ext': 'mp4', - 'upload_date': '20180814', - 'timestamp': 1534290623, - 'title': 'Pandorum', + 'title': 'Quarantäne', + 'description': 'md5:6af0373bd0fcc4f13e5d47701903d675', + 'episode': 'Episode 73', + 'episode_number': 73, + 'season': 'Season 4', + 'season_number': 4, + 'series': 'Stargate Atlantis', + 'upload_date': '20240525', + 'timestamp': 1716643200, + 'duration': 2503.2, + 'thumbnail': 'https://eu1-prod-images.disco-api.com/2024/05/21/c81fcb45-8902-309b-badb-4e6d546b575d.jpeg', + 'creators': ['Tele5'], + 'tags': [], }, - 'params': { - 'skip_download': True, - }, - 'skip': 'No longer available: "404 Seite nicht gefunden"', }, { - # jwplatform, nexx unavailable - 'url': 'https://www.tele5.de/filme/ghoul-das-geheimnis-des-friedhofmonsters/', + # only slug_a + 'url': 'https://tele5.de/mediathek/inside-out', 'info_dict': { - 'id': 'WJuiOlUp', + 'id': '6819502', 'ext': 'mp4', - 'upload_date': '20200603', - 'timestamp': 1591214400, - 'title': 'Ghoul - Das Geheimnis des Friedhofmonsters', - 'description': 'md5:42002af1d887ff3d5b2b3ca1f8137d97', + 'title': 'Inside out', + 'description': 'md5:7e5f32ed0be5ddbd27713a34b9293bfd', + 'series': 'Inside out', + 'upload_date': '20240523', + 'timestamp': 1716494400, + 'duration': 5343.4, + 'thumbnail': 'https://eu1-prod-images.disco-api.com/2024/05/15/181eba3c-f9f0-3faf-b14d-0097050a3aa4.jpeg', + 'creators': ['Tele5'], + 'tags': [], }, - 'params': { - 'skip_download': True, - }, - 'skip': 'No longer available, redirects to Filme page', }, { - 'url': 'https://tele5.de/mediathek/angel-of-mine/', + # playlist + 'url': 'https://tele5.de/mediathek/schlefaz', 'info_dict': { - 'id': '1252360', - 'ext': 'mp4', - 'upload_date': '20220109', - 'timestamp': 1641762000, - 'title': 'Angel of Mine', - 'description': 'md5:a72546a175e1286eb3251843a52d1ad7', + 'id': 'mediathek-schlefaz', }, - 'params': { - 'format': 'bestvideo', - }, - }, { - 'url': 'https://www.tele5.de/kalkofes-mattscheibe/video-clips/politik-und-gesellschaft?ve_id=1551191', - 'only_matching': True, - }, { - 'url': 'https://www.tele5.de/video-clip/?ve_id=1609440', - 'only_matching': True, - }, { - 'url': 'https://www.tele5.de/filme/schlefaz-dragon-crusaders/', - 'only_matching': True, - }, { - 'url': 'https://www.tele5.de/filme/making-of/avengers-endgame/', - 'only_matching': True, - }, { - 'url': 'https://www.tele5.de/star-trek/raumschiff-voyager/ganze-folge/das-vinculum/', - 'only_matching': True, - }, { - 'url': 'https://www.tele5.de/anders-ist-sevda/', - 'only_matching': True, + 'playlist_mincount': 3, }] def _real_extract(self, url): - video_id = self._match_id(url) - webpage = self._download_webpage(url, video_id) - player_element = self._search_regex(r'(]+?>)', webpage, 'video player') - player_info = extract_attributes(player_element) - asset_id, country, realm = (player_info[x] for x in ('assetid', 'locale', 'realm', )) - endpoint = compat_urlparse.urlparse(player_info['endpoint']).hostname - source_type = player_info.get('sourcetype') - if source_type: - endpoint = '%s-%s' % (source_type, endpoint) - try: - return self._get_disco_api_info(url, asset_id, endpoint, realm, country) - except ExtractorError as e: - if getattr(e, 'message', '') == 'Missing deviceId in context': - self.report_drm(video_id) - raise + parent_slug, slug_a, slug_b = self._match_valid_url(url).group('parent_slug', 'slug_a', 'slug_b') + playlist_id = join_nonempty(parent_slug, slug_a, slug_b, delim='-') + + query = {'environment': 'tele5', 'v': '2'} + if not slug_b: + endpoint = f'page/{slug_a}' + query['parent_slug'] = parent_slug + else: + endpoint = f'videos/{slug_b}' + query['filter[show.slug]'] = slug_a + cms_data = self._download_json(f'https://de-api.loma-cms.com/feloma/{endpoint}/', playlist_id, query=query) + + return self.playlist_result(map( + functools.partial(self._get_disco_api_info, url, disco_host='eu1-prod.disco-api.com', realm='dmaxde', country='DE'), + traverse_obj(cms_data, ('blocks', ..., 'videoId', {str}))), playlist_id) + + def _update_disco_api_headers(self, headers, disco_base, display_id, realm): + headers.update({ + 'x-disco-params': f'realm={realm}', + 'x-disco-client': 'Alps:HyogaPlayer:0.0.0', + 'Authorization': self._get_auth(disco_base, display_id, realm), + }) diff --git a/yt-dlp/yt_dlp/extractor/voot.py b/yt-dlp/yt_dlp/extractor/voot.py deleted file mode 100644 index ef77bedd27..0000000000 --- a/yt-dlp/yt_dlp/extractor/voot.py +++ /dev/null @@ -1,212 +0,0 @@ -import json -import time -import uuid - -from .common import InfoExtractor -from ..compat import compat_str -from ..networking.exceptions import HTTPError -from ..utils import ( - ExtractorError, - float_or_none, - int_or_none, - jwt_decode_hs256, - parse_age_limit, - traverse_obj, - try_call, - try_get, - unified_strdate, -) - - -class VootBaseIE(InfoExtractor): - _NETRC_MACHINE = 'voot' - _GEO_BYPASS = False - _LOGIN_HINT = 'Log in with "-u -p ", or use "-u token -p " to login with auth token.' - _TOKEN = None - _EXPIRY = 0 - _API_HEADERS = {'Origin': 'https://www.voot.com', 'Referer': 'https://www.voot.com/'} - - def _perform_login(self, username, password): - if self._TOKEN and self._EXPIRY: - return - - if username.lower() == 'token' and try_call(lambda: jwt_decode_hs256(password)): - VootBaseIE._TOKEN = password - VootBaseIE._EXPIRY = jwt_decode_hs256(password)['exp'] - self.report_login() - - # Mobile number as username is not supported - elif not username.isdigit(): - check_username = self._download_json( - 'https://userauth.voot.com/usersV3/v3/checkUser', None, data=json.dumps({ - 'type': 'email', - 'email': username - }, separators=(',', ':')).encode(), headers={ - **self._API_HEADERS, - 'Content-Type': 'application/json;charset=utf-8', - }, note='Checking username', expected_status=403) - if not traverse_obj(check_username, ('isExist', {bool})): - if traverse_obj(check_username, ('status', 'code', {int})) == 9999: - self.raise_geo_restricted(countries=['IN']) - raise ExtractorError('Incorrect username', expected=True) - auth_token = traverse_obj(self._download_json( - 'https://userauth.voot.com/usersV3/v3/login', None, data=json.dumps({ - 'type': 'traditional', - 'deviceId': str(uuid.uuid4()), - 'deviceBrand': 'PC/MAC', - 'data': { - 'email': username, - 'password': password - } - }, separators=(',', ':')).encode(), headers={ - **self._API_HEADERS, - 'Content-Type': 'application/json;charset=utf-8', - }, note='Logging in', expected_status=400), ('data', 'authToken', {dict})) - if not auth_token: - raise ExtractorError('Incorrect password', expected=True) - VootBaseIE._TOKEN = auth_token['accessToken'] - VootBaseIE._EXPIRY = auth_token['expirationTime'] - - else: - raise ExtractorError(self._LOGIN_HINT, expected=True) - - def _check_token_expiry(self): - if int(time.time()) >= self._EXPIRY: - raise ExtractorError('Access token has expired', expected=True) - - def _real_initialize(self): - if not self._TOKEN: - self.raise_login_required(self._LOGIN_HINT, method=None) - self._check_token_expiry() - - -class VootIE(VootBaseIE): - _WORKING = False - _VALID_URL = r'''(?x) - (?: - voot:| - https?://(?:www\.)?voot\.com/? - (?: - movies?/[^/]+/| - (?:shows|kids)/(?:[^/]+/){4} - ) - ) - (?P\d{3,}) - ''' - _TESTS = [{ - 'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/1/360558/is-this-the-end-of-kamini-/441353', - 'info_dict': { - 'id': '441353', - 'ext': 'mp4', - 'title': 'Is this the end of Kamini?', - 'description': 'md5:06291fbbbc4dcbe21235c40c262507c1', - 'timestamp': 1472103000, - 'upload_date': '20160825', - 'series': 'Ishq Ka Rang Safed', - 'season_number': 1, - 'episode': 'Is this the end of Kamini?', - 'episode_number': 340, - 'release_date': '20160825', - 'season': 'Season 1', - 'age_limit': 13, - 'duration': 1146.0, - }, - 'params': {'skip_download': 'm3u8'}, - }, { - 'url': 'https://www.voot.com/kids/characters/mighty-cat-masked-niyander-e-/400478/school-bag-disappears/440925', - 'only_matching': True, - }, { - 'url': 'https://www.voot.com/movies/pandavas-5/424627', - 'only_matching': True, - }, { - 'url': 'https://www.voot.com/movie/fight-club/621842', - 'only_matching': True, - }] - - def _real_extract(self, url): - video_id = self._match_id(url) - media_info = self._download_json( - 'https://psapi.voot.com/jio/voot/v1/voot-web/content/query/asset-details', video_id, - query={'ids': f'include:{video_id}', 'responseType': 'common'}, headers={'accesstoken': self._TOKEN}) - - try: - m3u8_url = self._download_json( - 'https://vootapi.media.jio.com/playback/v1/playbackrights', video_id, - 'Downloading playback JSON', data=b'{}', headers={ - **self.geo_verification_headers(), - **self._API_HEADERS, - 'Content-Type': 'application/json;charset=utf-8', - 'platform': 'androidwebdesktop', - 'vootid': video_id, - 'voottoken': self._TOKEN, - })['m3u8'] - except ExtractorError as e: - if isinstance(e.cause, HTTPError) and e.cause.status == 400: - self._check_token_expiry() - raise - - formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls') - self._remove_duplicate_formats(formats) - - return { - 'id': video_id, - # '/_definst_/smil:vod/' m3u8 manifests claim to have 720p+ formats but max out at 480p - 'formats': traverse_obj(formats, ( - lambda _, v: '/_definst_/smil:vod/' not in v['url'] or v['height'] <= 480)), - 'http_headers': self._API_HEADERS, - **traverse_obj(media_info, ('result', 0, { - 'title': ('fullTitle', {str}), - 'description': ('fullSynopsis', {str}), - 'series': ('showName', {str}), - 'season_number': ('season', {int_or_none}), - 'episode': ('fullTitle', {str}), - 'episode_number': ('episode', {int_or_none}), - 'timestamp': ('uploadTime', {int_or_none}), - 'release_date': ('telecastDate', {unified_strdate}), - 'age_limit': ('ageNemonic', {parse_age_limit}), - 'duration': ('duration', {float_or_none}), - })), - } - - -class VootSeriesIE(VootBaseIE): - _WORKING = False - _VALID_URL = r'https?://(?:www\.)?voot\.com/shows/[^/]+/(?P\d{3,})' - _TESTS = [{ - 'url': 'https://www.voot.com/shows/chakravartin-ashoka-samrat/100002', - 'playlist_mincount': 442, - 'info_dict': { - 'id': '100002', - }, - }, { - 'url': 'https://www.voot.com/shows/ishq-ka-rang-safed/100003', - 'playlist_mincount': 341, - 'info_dict': { - 'id': '100003', - }, - }] - _SHOW_API = 'https://psapi.voot.com/media/voot/v1/voot-web/content/generic/season-by-show?sort=season%3Aasc&id={}&responseType=common' - _SEASON_API = 'https://psapi.voot.com/media/voot/v1/voot-web/content/generic/series-wise-episode?sort=episode%3Aasc&id={}&responseType=common&page={:d}' - - def _entries(self, show_id): - show_json = self._download_json(self._SHOW_API.format(show_id), video_id=show_id) - for season in show_json.get('result', []): - page_num = 1 - season_id = try_get(season, lambda x: x['id'], compat_str) - season_json = self._download_json(self._SEASON_API.format(season_id, page_num), - video_id=season_id, - note='Downloading JSON metadata page %d' % page_num) - episodes_json = season_json.get('result', []) - while episodes_json: - page_num += 1 - for episode in episodes_json: - video_id = episode.get('id') - yield self.url_result( - 'voot:%s' % video_id, ie=VootIE.ie_key(), video_id=video_id) - episodes_json = self._download_json(self._SEASON_API.format(season_id, page_num), - video_id=season_id, - note='Downloading JSON metadata page %d' % page_num)['result'] - - def _real_extract(self, url): - show_id = self._match_id(url) - return self.playlist_result(self._entries(show_id), playlist_id=show_id) diff --git a/yt-dlp/yt_dlp/extractor/xiaohongshu.py b/yt-dlp/yt_dlp/extractor/xiaohongshu.py new file mode 100644 index 0000000000..faad9d9235 --- /dev/null +++ b/yt-dlp/yt_dlp/extractor/xiaohongshu.py @@ -0,0 +1,83 @@ +import functools + +from .common import InfoExtractor +from ..utils import ( + float_or_none, + int_or_none, + js_to_json, + url_or_none, +) +from ..utils.traversal import traverse_obj + + +class XiaoHongShuIE(InfoExtractor): + _VALID_URL = r'https?://www\.xiaohongshu\.com/explore/(?P[\da-f]+)' + IE_DESC = '小红书' + _TESTS = [{ + 'url': 'https://www.xiaohongshu.com/explore/6411cf99000000001300b6d9', + 'md5': '2a87a77ddbedcaeeda8d7eae61b61228', + 'info_dict': { + 'id': '6411cf99000000001300b6d9', + 'ext': 'mp4', + 'uploader_id': '5c31698d0000000007018a31', + 'description': '#今日快乐今日发[话题]# #吃货薯看这里[话题]# #香妃蛋糕[话题]# #小五卷蛋糕[话题]# #新手蛋糕卷[话题]#', + 'title': '香妃蛋糕也太香了吧🔥不需要卷❗️绝对的友好', + 'tags': ['今日快乐今日发', '吃货薯看这里', '香妃蛋糕', '小五卷蛋糕', '新手蛋糕卷'], + 'duration': 101.726, + 'thumbnail': r're:https?://sns-webpic-qc\.xhscdn\.com/\d+/[a-z0-9]+/[\w]+', + } + }] + + def _real_extract(self, url): + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + initial_state = self._search_json( + r'window\.__INITIAL_STATE__\s*=', webpage, 'initial state', display_id, transform_source=js_to_json) + + note_info = traverse_obj(initial_state, ('note', 'noteDetailMap', display_id, 'note')) + video_info = traverse_obj(note_info, ('video', 'media', 'stream', ('h264', 'av1', 'h265'), ...)) + + formats = [] + for info in video_info: + format_info = traverse_obj(info, { + 'fps': ('fps', {int_or_none}), + 'width': ('width', {int_or_none}), + 'height': ('height', {int_or_none}), + 'vcodec': ('videoCodec', {str}), + 'acodec': ('audioCodec', {str}), + 'abr': ('audioBitrate', {int_or_none}), + 'vbr': ('videoBitrate', {int_or_none}), + 'audio_channels': ('audioChannels', {int_or_none}), + 'tbr': ('avgBitrate', {int_or_none}), + 'format': ('qualityType', {str}), + 'filesize': ('size', {int_or_none}), + 'duration': ('duration', {functools.partial(float_or_none, scale=1000)}) + }) + + formats.extend(traverse_obj(info, (('mediaUrl', ('backupUrls', ...)), { + lambda u: url_or_none(u) and {'url': u, **format_info}}))) + + thumbnails = [] + for image_info in traverse_obj(note_info, ('imageList', ...)): + thumbnail_info = traverse_obj(image_info, { + 'height': ('height', {int_or_none}), + 'width': ('width', {int_or_none}), + }) + for thumb_url in traverse_obj(image_info, (('urlDefault', 'urlPre'), {url_or_none})): + thumbnails.append({ + 'url': thumb_url, + **thumbnail_info, + }) + + return { + 'id': display_id, + 'formats': formats, + 'thumbnails': thumbnails, + 'title': self._html_search_meta(['og:title'], webpage, default=None), + **traverse_obj(note_info, { + 'title': ('title', {str}), + 'description': ('desc', {str}), + 'tags': ('tagList', ..., 'name', {str}), + 'uploader_id': ('user', 'userId', {str}), + }), + }