Update On Mon Dec 2 19:39:24 CET 2024

This commit is contained in:
github-action[bot]
2024-12-02 19:39:25 +01:00
parent c3e68f49c1
commit 1bb062325d
280 changed files with 7754 additions and 5538 deletions

View File

@@ -104,7 +104,9 @@ ninja yass
## macOS
1. Make sure you have both of [Xcode] and [Homebrew] installed:
1. Make sure you have [Xcode] installed:
After download _Xcode.xip_ and extract _Xcode.app_ into `/Applications` (optional)
Run in `Terminal`:
```
@@ -113,18 +115,17 @@ xcode-select --install
xcodebuild -runFirstLaunch
```
Run in `Terminal`:
```
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
```
2. Install the required build tools via homebrew
2. Make sure you have [MacPorts] installed:
Download pkg file from [MacPorts website][MacPorts]. Open and install it.
Run in `Terminal`:
```
brew install ninja cmake go p7zip
sudo port selfupdate
sudo port install ninja cmake go p7zip
```
3. Compile the program with Release configuration.
3. Compile the program with _Release_ configuration.
```
git clone https://github.com/Chilledheart/yass
cd yass
@@ -261,7 +262,7 @@ ninja yass
[GCC]: https://gcc.gnu.org/
[NASM]: https://www.nasm.us/
[Xcode]: https://apps.apple.com/us/app/xcode/id497799835?mt=12
[HomeBrew]: https://docs.brew.sh/Installation
[MacPorts]: https://guide.macports.org/chunked/installing.macports.html
[llvm-win64]: https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.8/LLVM-18.1.8-win64.exe
[msys2]: https://www.msys2.org/
[EPEL]: https://docs.fedoraproject.org/en-US/epel

View File

@@ -5094,6 +5094,7 @@ if (BUILD_TESTS)
src/core/process_utils_test.cpp
src/core/utils_test.cpp
src/net/asio_ssl_test.cpp
src/net/io_queue_test.cpp
src/net/cipher_test.cpp
src/net/c-ares_test.cpp
src/net/padding_test.cpp

View File

@@ -1312,8 +1312,11 @@ void CliConnection::WriteUpstreamAuthRequest() {
std::string username = absl::GetFlag(FLAGS_username);
std::string password = absl::GetFlag(FLAGS_password);
ByteRange req(reinterpret_cast<const uint8_t*>(&header), sizeof(header));
std::shared_ptr<IOBuf> buf = IOBuf::copyBuffer(req);
std::shared_ptr<IOBuf> buf = upstream_.front();
buf->reserve(0, sizeof(header));
memcpy(buf->mutable_tail(), &header, sizeof(header));
buf->append(sizeof(header));
buf->reserve(0, sizeof(uint8_t));
*buf->mutable_tail() = username.size();
buf->append(sizeof(uint8_t));
@@ -1330,7 +1333,6 @@ void CliConnection::WriteUpstreamAuthRequest() {
memcpy(buf->mutable_tail(), password.c_str(), password.size());
buf->append(password.size());
upstream_.replace_front(buf);
WriteUpstreamInPipe();
}
@@ -1447,8 +1449,10 @@ void CliConnection::WriteUpstreamSocks5Request() {
header.command = socks5::cmd_connect;
header.null_byte = 0;
ByteRange req(reinterpret_cast<const uint8_t*>(&header), sizeof(header));
std::shared_ptr<IOBuf> buf = IOBuf::copyBuffer(req);
std::shared_ptr<IOBuf> buf = upstream_.front();
buf->reserve(0, sizeof(header));
memcpy(buf->mutable_tail(), &header, sizeof(header));
buf->append(sizeof(header));
span<const uint8_t> address;
std::string domain_name;
@@ -1492,7 +1496,6 @@ void CliConnection::WriteUpstreamSocks5Request() {
socks_handshake_ = true;
upstream_.replace_front(buf);
WriteUpstreamInPipe();
}

View File

@@ -12,7 +12,7 @@
namespace net {
template <typename X = IOBuf,
int DEFAULT_QUEUE_LENGTH =
unsigned int DEFAULT_QUEUE_LENGTH =
#if BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_IOS) || BUILDFLAG(IS_OHOS) || defined(__MUSL__)
8
#else
@@ -22,58 +22,66 @@ template <typename X = IOBuf,
class IoQueue {
using T = std::shared_ptr<X>;
using Vector = absl::InlinedVector<T, DEFAULT_QUEUE_LENGTH>;
static_assert(DEFAULT_QUEUE_LENGTH >= 2, "Default Queue Depth is too small");
public:
IoQueue() { queue_.resize(DEFAULT_QUEUE_LENGTH); }
using size_type = Vector::size_type;
public:
IoQueue() { DCHECK_EQ(DEFAULT_QUEUE_LENGTH, queue_.size()); }
IoQueue(const IoQueue&) = delete;
IoQueue& operator=(const IoQueue&) = delete;
IoQueue(IoQueue&& rhs) {
idx_ = rhs.idx_;
end_idx_ = rhs.end_idx_;
queue_.resize(DEFAULT_QUEUE_LENGTH);
DCHECK_EQ(DEFAULT_QUEUE_LENGTH, queue_.size());
DCHECK_LE(DEFAULT_QUEUE_LENGTH, rhs.queue_.size());
std::swap(queue_, rhs.queue_);
dirty_front_ = rhs.dirty_front_;
rhs.idx_ = {};
rhs.end_idx_ = {};
rhs.dirty_front_ = {};
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
DCHECK(rhs.empty());
#if DCHECK_IS_ON()
for (auto buf : rhs.queue_) {
DCHECK(!buf);
}
#endif
DCHECK_EQ(DEFAULT_QUEUE_LENGTH, rhs.queue_.size());
}
IoQueue& operator=(IoQueue&& rhs) {
idx_ = rhs.idx_;
end_idx_ = rhs.end_idx_;
DCHECK(queue_.size());
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
DCHECK_LE(DEFAULT_QUEUE_LENGTH, rhs.queue_.size());
std::swap(queue_, rhs.queue_);
dirty_front_ = rhs.dirty_front_;
rhs.idx_ = {};
rhs.end_idx_ = {};
rhs.dirty_front_ = {};
// better way to optimize below code
// rhs.queue_.clear();
// rhs.queue_.resize(DEFAULT_QUEUE_LENGTH);
Vector empty_queue{DEFAULT_QUEUE_LENGTH};
std::swap(empty_queue, rhs.queue_);
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
DCHECK(rhs.empty());
#if DCHECK_IS_ON()
for (auto buf : rhs.queue_) {
DCHECK(!buf);
}
#endif
DCHECK_EQ(DEFAULT_QUEUE_LENGTH, rhs.queue_.size());
return *this;
}
bool empty() const { return idx_ == end_idx_; }
void replace_front(T buf) {
DCHECK(!empty());
dirty_front_ = true;
queue_[idx_] = buf;
bool empty() const {
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
if (idx_ == end_idx_) {
#if DCHECK_IS_ON()
for (auto buf : queue_) {
DCHECK(!buf);
}
#endif
return true;
}
return false;
}
void push_back(T buf) {
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
DCHECK(buf);
queue_[end_idx_] = buf;
end_idx_ = (end_idx_ + 1) % queue_.size();
if (end_idx_ == idx_) {
LOG(INFO) << "Current IO queue is full, enlarging by 2x to " << 2 * queue_.size();
VLOG(1) << "Current IO queue is full, enlarging by 2x to " << 2 * queue_.size();
enlarge_queue_by_2x();
}
}
@@ -82,25 +90,30 @@ class IoQueue {
T front() {
DCHECK(!empty());
dirty_front_ = true;
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
return queue_[idx_];
}
void pop_front() {
DCHECK(!empty());
dirty_front_ = false;
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
queue_[idx_] = nullptr;
idx_ = (idx_ + 1) % queue_.size();
}
T back() {
DCHECK(!empty());
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
return queue_[(end_idx_ + queue_.size() - 1) % queue_.size()];
}
size_t length() const { return (end_idx_ + queue_.size() - idx_) % queue_.size(); }
size_type length() const {
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
return (end_idx_ + queue_.size() - idx_) % queue_.size();
}
size_t byte_length() const {
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
if (empty()) {
return 0u;
}
@@ -110,14 +123,30 @@ class IoQueue {
return ret;
}
void clear() { *this = IoQueue(); }
void clear() {
DCHECK_LE(DEFAULT_QUEUE_LENGTH, queue_.size());
*this = IoQueue();
DCHECK_EQ(DEFAULT_QUEUE_LENGTH, queue_.size());
DCHECK(empty());
}
void swap(IoQueue& other) {
if (this != std::addressof(other)) {
std::swap(idx_, other.idx_);
std::swap(end_idx_, other.end_idx_);
std::swap(queue_, other.queue_);
}
}
private:
void enlarge_queue_by_2x() {
DCHECK(queue_.size());
DCHECK_LE(queue_.size(), 32u << 10);
DCHECK_EQ(idx_, end_idx_);
DCHECK_LE(queue_.size() << 2, static_cast<size_type>(INT_MAX)) << "index overflow";
Vector new_queue;
DCHECK_EQ(0u, new_queue.size());
new_queue.reserve(queue_.size() << 1);
DCHECK_LE(queue_.size() << 1, new_queue.capacity());
if (idx_ < end_idx_) {
new_queue.insert(new_queue.end(), queue_.begin() + idx_, queue_.begin() + end_idx_);
} else /* if (idx_ >= end_idx_) */ {
@@ -127,16 +156,22 @@ class IoQueue {
idx_ = 0;
end_idx_ = queue_.size();
new_queue.resize(queue_.size() << 1);
DCHECK_EQ(queue_.size() << 1, new_queue.size());
std::swap(queue_, new_queue);
DCHECK_EQ(new_queue.size() << 1, queue_.size());
}
private:
int idx_ = 0;
int end_idx_ = 0;
Vector queue_;
bool dirty_front_ = false;
Vector queue_{static_cast<size_type>(DEFAULT_QUEUE_LENGTH)};
};
template <typename X, unsigned int Q>
void swap(IoQueue<X, Q>& lhs, IoQueue<X, Q>& rhs) {
lhs.swap(rhs);
}
} // namespace net
#endif // CORE_IO_QUEUE_HPP

View File

@@ -0,0 +1,335 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2024 Chilledheart */
#include <build/build_config.h>
#include <gtest/gtest.h>
#include <vector>
#include "core/utils.hpp"
#include "net/io_queue.hpp"
#include "net/iobuf.hpp"
using namespace net;
static constexpr unsigned int kDefaultDepth = 8u;
static constexpr unsigned int kBufferSize = 4096u;
static constexpr const char kBuffer[kBufferSize] = {};
TEST(IoQueueTest, Construct) {
IoQueue<IOBuf, kDefaultDepth> queue;
ASSERT_TRUE(queue.empty());
}
TEST(IoQueueTest, PushBackAndPopFrontVariant0) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
queue.push_back(kBuffer, kBufferSize);
}
ASSERT_EQ(kDefaultDepth, queue.length());
ASSERT_EQ(kDefaultDepth * kBufferSize, queue.byte_length());
for (unsigned int i = 0u; i < kDefaultDepth / 2; ++i) {
queue.pop_front();
}
ASSERT_EQ(kDefaultDepth / 2, queue.length());
ASSERT_EQ(kDefaultDepth / 2 * kBufferSize, queue.byte_length());
for (unsigned int i = 0u; i < kDefaultDepth / 2; ++i) {
queue.pop_front();
}
ASSERT_TRUE(queue.empty());
}
TEST(IoQueueTest, PushBackAndPopFrontVariant1) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
queue.push_back(IOBuf::copyBuffer(std::string(kBuffer, kBufferSize)));
}
ASSERT_EQ(kDefaultDepth, queue.length());
ASSERT_EQ(kDefaultDepth * kBufferSize, queue.byte_length());
for (unsigned int i = 0u; i < kDefaultDepth / 2; ++i) {
queue.pop_front();
}
ASSERT_EQ(kDefaultDepth / 2, queue.length());
ASSERT_EQ(kDefaultDepth / 2 * kBufferSize, queue.byte_length());
for (unsigned int i = 0u; i < kDefaultDepth / 2; ++i) {
queue.pop_front();
}
ASSERT_TRUE(queue.empty());
}
TEST(IoQueueTest, MoveConstruct) {
IoQueue<IOBuf, kDefaultDepth * 2> pending_data;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
pending_data.push_back(kBuffer, kBufferSize);
}
auto queue = std::move(pending_data);
ASSERT_TRUE(pending_data.empty());
ASSERT_EQ(kDefaultDepth, queue.length());
}
TEST(IoQueueTest, MoveConstructOverInlinedStorage) {
IoQueue<IOBuf, kDefaultDepth * 2> pending_data;
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
pending_data.push_back(kBuffer, kBufferSize);
}
auto queue = std::move(pending_data);
ASSERT_TRUE(pending_data.empty());
ASSERT_EQ(kDefaultDepth * 2, queue.length());
}
TEST(IoQueueTest, MoveAssignment) {
IoQueue<IOBuf, kDefaultDepth * 2> queue, pending_data;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
queue.push_back(kBuffer, kBufferSize);
}
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
pending_data.push_back(kBuffer, kBufferSize);
}
queue = std::move(pending_data);
ASSERT_TRUE(pending_data.empty());
ASSERT_EQ(kDefaultDepth, queue.length());
}
TEST(IoQueueTest, MoveAssignmentLhsOverInlinedStorage) {
IoQueue<IOBuf, kDefaultDepth * 2> pending_data, queue;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
pending_data.push_back(kBuffer, kBufferSize);
}
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
queue.push_back(kBuffer, kBufferSize);
}
queue = std::move(pending_data);
ASSERT_TRUE(pending_data.empty());
ASSERT_EQ(kDefaultDepth, queue.length());
}
TEST(IoQueueTest, MoveAssignmentRhsOverInlinedStorage) {
IoQueue<IOBuf, kDefaultDepth * 2> pending_data, queue;
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
pending_data.push_back(kBuffer, kBufferSize);
}
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
queue.push_back(kBuffer, kBufferSize);
}
queue = std::move(pending_data);
ASSERT_TRUE(pending_data.empty());
ASSERT_EQ(kDefaultDepth * 2, queue.length());
}
TEST(IoQueueTest, MoveAssignmentBothOverInlinedStorage) {
IoQueue<IOBuf, kDefaultDepth * 2> pending_data, queue;
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
pending_data.push_back(kBuffer, kBufferSize);
}
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
queue.push_back(kBuffer, kBufferSize);
}
queue = std::move(pending_data);
ASSERT_TRUE(pending_data.empty());
ASSERT_EQ(kDefaultDepth * 2, queue.length());
}
TEST(IoQueueTest, Clear) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
queue.push_back(kBuffer, kBufferSize);
}
queue.clear();
ASSERT_TRUE(queue.empty());
ASSERT_EQ(0u, queue.length());
ASSERT_EQ(0u, queue.byte_length());
}
TEST(IoQueueTest, ClearOverInlinedStorage) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
queue.push_back(kBuffer, kBufferSize);
}
queue.clear();
ASSERT_TRUE(queue.empty());
ASSERT_EQ(0u, queue.length());
}
TEST(IoQueueTest, SwapEmptyWith) {
IoQueue<IOBuf, kDefaultDepth * 2> queue, empty_queue;
ASSERT_TRUE(queue.empty());
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
empty_queue.push_back(kBuffer, kBufferSize);
}
std::swap(queue, empty_queue);
ASSERT_TRUE(empty_queue.empty());
ASSERT_EQ(0u, empty_queue.length());
ASSERT_EQ(kDefaultDepth, queue.length());
ASSERT_EQ(kDefaultDepth * kBufferSize, queue.byte_length());
}
TEST(IoQueueTest, SwapWithEmpty) {
IoQueue<IOBuf, kDefaultDepth * 2> queue, empty_queue;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
empty_queue.push_back(kBuffer, kBufferSize);
}
std::swap(empty_queue, queue);
ASSERT_TRUE(empty_queue.empty());
ASSERT_EQ(0u, empty_queue.length());
ASSERT_EQ(kDefaultDepth, queue.length());
ASSERT_EQ(kDefaultDepth * kBufferSize, queue.byte_length());
}
TEST(IoQueueTest, SwapNonEmpty) {
IoQueue<IOBuf, kDefaultDepth * 10> lhs, rhs;
for (unsigned int i = 0u; i < kDefaultDepth * 3; ++i) {
lhs.push_back(kBuffer, kBufferSize * 5);
}
for (unsigned int i = 0u; i < kDefaultDepth * 7; ++i) {
rhs.push_back(kBuffer, kBufferSize * 9);
}
std::swap(lhs, rhs);
ASSERT_EQ(7 * kDefaultDepth, lhs.length());
ASSERT_EQ(7 * 9 * kDefaultDepth * kBufferSize, lhs.byte_length());
ASSERT_EQ(3 * kDefaultDepth, rhs.length());
ASSERT_EQ(3 * 5 * kDefaultDepth * kBufferSize, rhs.byte_length());
}
TEST(IoQueueTest, EnlargeVariant0) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
ASSERT_TRUE(queue.empty());
std::vector<std::shared_ptr<IOBuf>> v;
// push idx_ to kDefaultDepth - 1
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
queue.push_back(kBuffer, kBufferSize);
queue.pop_front();
}
ASSERT_TRUE(queue.empty());
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
std::shared_ptr<IOBuf> buf = IOBuf::copyBuffer(kBuffer, kBufferSize);
v.push_back(buf);
queue.push_back(buf);
}
ASSERT_EQ(kDefaultDepth * 2, v.size());
ASSERT_EQ(kDefaultDepth * 2, queue.length());
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
auto buf = queue.front();
queue.pop_front();
EXPECT_EQ(v[i].get(), buf.get());
}
ASSERT_TRUE(queue.empty());
}
TEST(IoQueueTest, EnlargeVariant1) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
ASSERT_TRUE(queue.empty());
std::vector<std::shared_ptr<IOBuf>> v;
// push idx_ to 0
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
std::shared_ptr<IOBuf> buf = IOBuf::copyBuffer(kBuffer, kBufferSize);
v.push_back(buf);
queue.push_back(buf);
}
ASSERT_EQ(kDefaultDepth * 2, v.size());
ASSERT_EQ(kDefaultDepth * 2, queue.length());
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
auto buf = queue.front();
queue.pop_front();
EXPECT_EQ(v[i].get(), buf.get());
}
ASSERT_TRUE(queue.empty());
}
TEST(IoQueueTest, EnlargeVariant2) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
ASSERT_TRUE(queue.empty());
std::vector<std::shared_ptr<IOBuf>> v;
// push idx_ to kDefaultDepth * 2 - 1
for (unsigned int i = 0u; i < kDefaultDepth * 2 - 1; ++i) {
queue.push_back(kBuffer, kBufferSize);
queue.pop_front();
}
// ASSERT_EQ(queue.end_idx_, queue.length() - 1);
ASSERT_TRUE(queue.empty());
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
std::shared_ptr<IOBuf> buf = IOBuf::copyBuffer(kBuffer, kBufferSize);
v.push_back(buf);
queue.push_back(buf);
}
ASSERT_EQ(kDefaultDepth * 2, v.size());
ASSERT_EQ(kDefaultDepth * 2, queue.length());
for (unsigned int i = 0u; i < kDefaultDepth * 2; ++i) {
auto buf = queue.front();
queue.pop_front();
EXPECT_EQ(v[i].get(), buf.get());
}
ASSERT_TRUE(queue.empty());
}
TEST(IoQueueTest, EnlargeTwice) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
ASSERT_TRUE(queue.empty());
std::vector<std::shared_ptr<IOBuf>> v;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
queue.push_back(kBuffer, kBufferSize);
queue.pop_front();
}
for (unsigned int i = 0u; i < kDefaultDepth * 4; ++i) {
std::shared_ptr<IOBuf> buf = IOBuf::copyBuffer(kBuffer, kBufferSize);
v.push_back(buf);
queue.push_back(buf);
}
ASSERT_EQ(kDefaultDepth * 4, v.size());
ASSERT_EQ(kDefaultDepth * 4, queue.length());
for (unsigned int i = 0u; i < kDefaultDepth * 4; ++i) {
auto buf = queue.front();
queue.pop_front();
EXPECT_EQ(v[i].get(), buf.get());
}
ASSERT_TRUE(queue.empty());
}
TEST(IoQueueTest, EnlargeThird) {
IoQueue<IOBuf, kDefaultDepth * 2> queue;
ASSERT_TRUE(queue.empty());
std::vector<std::shared_ptr<IOBuf>> v;
for (unsigned int i = 0u; i < kDefaultDepth; ++i) {
queue.push_back(kBuffer, kBufferSize);
queue.pop_front();
}
for (unsigned int i = 0u; i < kDefaultDepth * 8; ++i) {
std::shared_ptr<IOBuf> buf = IOBuf::copyBuffer(kBuffer, kBufferSize);
v.push_back(buf);
queue.push_back(buf);
}
ASSERT_EQ(kDefaultDepth * 8, v.size());
ASSERT_EQ(kDefaultDepth * 8, queue.length());
for (unsigned int i = 0u; i < kDefaultDepth * 8; ++i) {
auto buf = queue.front();
queue.pop_front();
EXPECT_EQ(v[i].get(), buf.get());
}
ASSERT_TRUE(queue.empty());
}