drashupkin/phpclickhouse-laravel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v2.0.0 ClickHouse cluster support

Browse Source
This commit is contained in:
Denis Glushkov
2024-07-26 20:34:49 +05:00
parent 884b21f65f
commit 9a13f23674
20 changed files with 632 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+10
View File
@@ -1,3 +1,13 @@
## 2.0.0 [2024-07-26]
### Features
1. ClickHouse cluster support
### Breaking changes
1. The minimum required PHP version is 8.0
2. Removed deprecated method BaseModel::insert, use BaseModel::insertBulk instead
3. Method BaseModel::prepareAndInsert is marked as deprecated, use BaseModel::prepareAndInsertBulk instead
## 1.19.0 [2023-09-28]
### Features
README.md
+84 -5
View File
@@ -9,13 +9,13 @@ Adapter to Laravel and Lumen of the most popular libraries:
## Features
No dependency, only Curl (support php >=7.1 )
No dependency, only Curl (support php >=8.0 )
More: https://github.com/smi2/phpClickHouse#features
## Prerequisites
- PHP 7.1, 8.0
- PHP 8.0
- Laravel/Lumen 7+
- Clickhouse server
@@ -87,7 +87,6 @@ More about `$db` see here: https://github.com/smi2/phpClickHouse/blob/master/REA
```php
<?php
namespace App\Models\Clickhouse;
use PhpClickHouseLaravel\BaseModel;
@@ -105,7 +104,6 @@ class MyTable extends BaseModel
```php
<?php
class CreateMyTable extends \PhpClickHouseLaravel\Migration
{
/**
@@ -354,7 +352,6 @@ MyTable::insertAssoc([[1, 'str', new InsertArray(['a','b'])]]);
```php
<?php
namespace App\Models\Clickhouse;
use PhpClickHouseLaravel\BaseModel;
@@ -386,3 +383,85 @@ return new class extends \PhpClickHouseLaravel\Migration
}
};
```
### Cluster mode
**Important!**
* Each ClickHouse node must have one database name and login and password.
* For reading and writing, the connection is made to the first available node.
* Migrations executes on all nodes. If one of the nodes is unavailable, the migration will throw an exception.
Your config/database.php should look like:
```php
'clickhouse' => [
'driver' => 'clickhouse',
'cluster' => [
[
'host' => 'clickhouse01',
'port' => '8123',
],
[
'host' => 'clickhouse02',
'port' => '8123',
],
],
'database' => env('CLICKHOUSE_DATABASE','default'),
'username' => env('CLICKHOUSE_USERNAME','default'),
'password' => env('CLICKHOUSE_PASSWORD',''),
'timeout_connect' => env('CLICKHOUSE_TIMEOUT_CONNECT',2),
'timeout_query' => env('CLICKHOUSE_TIMEOUT_QUERY',2),
'https' => (bool)env('CLICKHOUSE_HTTPS', null),
'retries' => env('CLICKHOUSE_RETRIES', 0),
'settings' => [ // optional
'max_partitions_per_insert_block' => 300,
],
],
```
Migration is:
```php
<?php
return new class extends \PhpClickHouseLaravel\Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
static::write('
CREATE TABLE my_table (
id UInt32,
created_at DateTime,
field_one String,
field_two Int32
)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/default.my_table', '{replica}')
ORDER BY (id)
');
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
static::write('DROP TABLE my_table');
}
};
```
You can get the host of the current node and switch the active connection to the next node:
```php
$row = new MyTable();
echo $row->getThisClient()->getConnectHost();
// will print 'clickhouse01'
$row->resolveConnection()->getCluster()->slideNode();
echo $row->getThisClient()->getConnectHost();
// will print 'clickhouse02'
```
composer.json
+1 -1
View File
@@ -17,7 +17,7 @@
}
],
"require": {
"php": ">=7.1.0|>=8.0",
"php": ">=8.0",
"smi2/phpclickhouse": "^1.4.2",
"the-tinderbox/clickhouse-builder": "^6.0",
"illuminate/support": ">=7",
docker-compose.test.yaml
+18 -14
View File
@@ -5,25 +5,29 @@ services:
php:
build: tests/docker
depends_on:
- clickhouse
- clickhouse2
- clickhouse01
- clickhouse02
- zookeeper
volumes:
- ./:/src
clickhouse:
clickhouse01:
image: yandex/clickhouse-server
ulimits:
nproc: 65535
nofile:
soft: 262144
hard: 262144
ports:
- "18123:8123"
depends_on:
- zookeeper
volumes:
- ./tests/docker/clickhouse01:/etc/clickhouse-server
clickhouse2:
clickhouse02:
image: yandex/clickhouse-server
ulimits:
nproc: 65535
nofile:
soft: 262144
hard: 262144
ports:
- "18124:8123"
depends_on:
- zookeeper
volumes:
- ./tests/docker/clickhouse02:/etc/clickhouse-server
zookeeper:
image: zookeeper:3.7
src/BaseModel.php
+18 -18
View File
@@ -72,12 +72,12 @@ class BaseModel
* @var string
*/
protected $connection = Connection::DEFAULT_NAME;
/**
* Determine if an attribute or relation exists on the model.
* The __isset magic method is triggered by calling isset() or empty() on inaccessible properties.
*
* @param string $key The name of the attribute or relation.
* @param string $key The name of the attribute or relation.
* @return bool True if the attribute or relation exists, false otherwise.
*/
public function __isset($key)
@@ -97,7 +97,7 @@ class BaseModel
return false;
}
/**
* Get the table associated with the model.
*
@@ -190,18 +190,6 @@ class BaseModel
return $this->exists;
}
/**
* Bulk insert into Clickhouse database
* @param array[] $rows
* @return Statement
* @deprecated use insertBulk
*/
public static function insert(array $rows): Statement
{
$instance = new static();
return $instance->getThisClient()->insert($instance->getTableForInserts(), $rows);
}
/**
* Bulk insert into Clickhouse database
* @param array[] $rows
@@ -232,6 +220,21 @@ class BaseModel
* @param array $columns
* @return Statement
*/
public static function prepareAndInsertBulk(array $rows, array $columns = []): Statement
{
return static::insertBulk(
array_map('static::prepareFromRequest', $rows, $columns),
$columns
);
}
/**
* Prepare each row by calling static::prepareFromRequest to bulk insert into database
* @param array[] $rows
* @param array $columns
* @return Statement
* @deprecated use prepareAndInsertBulk
*/
public static function prepareAndInsert(array $rows, array $columns = []): Statement
{
$rows = array_map('static::prepareFromRequest', $rows, $columns);
@@ -319,9 +322,6 @@ class BaseModel
return $instance->newQuery()->select($select)->from($instance->getTable());
}
/**
* @return Builder
*/
protected function newQuery(): Builder
{
return new Builder($this->getThisClient());
src/Cluster.php
+94
View File
@@ -0,0 +1,94 @@
<?php
namespace PhpClickHouseLaravel;
use ClickHouseDB\Client;
use ClickHouseDB\Exception\TransportException;
use ClickHouseDB\Statement;
class Cluster
{
/**
* @var Client[]
*/
protected array $nodes;
protected int $activeNodeIndex;
public function __construct(
protected array $nodeConfigs
) {
foreach ($this->nodeConfigs as $index => $nodeConfig) {
try {
$this->nodes[$index] = static::createClient($nodeConfig);
$this->nodes[$index]->ping(true);
$this->activeNodeIndex = $index;
break;
} catch (TransportException $e) {
}
}
if (!isset($this->activeNodeIndex)) {
throw $e ?? new TransportException('No nodes are available');
}
}
public function write(string $sql, array $bindings = [], bool $exception = true): ?Statement
{
foreach ($this->nodeConfigs as $index => $config) {
if (empty($this->nodes[$index])) {
$this->nodes[$index] = static::createClient($config);
}
$statement = $this->nodes[$index]->write($sql, $bindings, $exception);
}
return $statement ?? null;
}
public function getActiveNode(): Client
{
return $this->nodes[$this->activeNodeIndex];
}
/**
* Switch active node to the next available node
* @return void
*/
public function slideNode(): void
{
$configCount = count($this->nodeConfigs);
if ($configCount < 2) {
return;
}
for ($i = 0; $i < $configCount; $i++) {
$nextIndex = $this->activeNodeIndex + 1;
if ($configCount == $nextIndex) {
$nextIndex = 0;
}
try {
$this->nodes[$nextIndex] ??= static::createClient($this->nodeConfigs[$nextIndex]);
$this->nodes[$nextIndex]->ping(true);
$this->activeNodeIndex = $nextIndex;
break;
} catch (TransportException) {
}
}
}
protected static function createClient(array $config): Client
{
$client = new Client($config);
$client->database($config['database']);
$client->setTimeout((int)$config['timeout_query']);
$client->setConnectTimeOut((int)$config['timeout_connect']);
if ($configSettings =& $config['settings']) {
$settings = $client->settings();
foreach ($configSettings as $sName => $sValue) {
$settings->set($sName, $sValue);
}
}
if ($retries = (int)($config['retries'] ?? null)) {
$curler = new CurlerRollingWithRetries();
$curler->setRetries($retries);
$client->transport()->setDirtyCurler($curler);
}
return $client;
}
}
src/Connection.php
+17 -25
View File
@@ -10,43 +10,36 @@ use Illuminate\Database\Connection as BaseConnection;
class Connection extends BaseConnection
{
public const DEFAULT_NAME = 'clickhouse';
/** @var Client */
protected $client;
protected Cluster $cluster;
public function getCluster(): Cluster
{
return $this->cluster;
}
/**
* @return Client
*/
public function getClient(): Client
{
return $this->client;
return $this->cluster->getActiveNode();
}
/**
* @param array $config
* @return static
*/
public static function createWithClient(array $config)
public static function createWithClient(array $config): self
{
$conn = new static(null, $config['database'], '', $config);
$conn->client = new Client($config);
$conn->client->database($config['database']);
$conn->client->setTimeout((int)$config['timeout_query']);
$conn->client->setConnectTimeOut((int)$config['timeout_connect']);
if ($configSettings =& $config['settings']) {
$settings = $conn->getClient()->settings();
foreach ($configSettings as $sName => $sValue) {
$settings->set($sName, $sValue);
$nodeConfigs = [];
if ($cluster = $config['cluster'] ?? null) {
foreach ($cluster as $node) {
$nodeConfigs[] = $node + $config;
}
} else {
$nodeConfigs[] = $config;
}
if ($retries = (int)($config['retries'] ?? null)) {
$curler = new CurlerRollingWithRetries();
$curler->setRetries($retries);
$conn->client->transport()->setDirtyCurler($curler);
}
$conn->cluster = new Cluster($nodeConfigs);
return $conn;
}
@@ -76,9 +69,8 @@ class Connection extends BaseConnection
public function select($query, $bindings = [], $useReadPdo = true): array
{
$query = QueryGrammar::prepareParameters($query);
return $this->run($query, $bindings, function ($query, $bindings) {
return $this->client->select($query, $bindings)->rows();
return $this->cluster->getActiveNode()->select($query, $bindings)->rows();
});
}
@@ -86,7 +78,7 @@ class Connection extends BaseConnection
public function statement($query, $bindings = []): bool
{
return $this->run($query, $bindings, function ($query, $bindings) {
return !$this->client->write($query, $bindings)->isError();
return !$this->cluster->getActiveNode()->write($query, $bindings)->isError();
});
}
src/Migration.php
+1 -1
View File
@@ -33,6 +33,6 @@ class Migration extends BaseMigration
return new Statement(new CurlerRequest());
}
}
return $instance->getThisClient()->write($sql, $bindings);
return $instance->resolveConnection()->getCluster()->write($sql, $bindings);
}
}
src/WithClient.php
+5
View File
@@ -20,4 +20,9 @@ trait WithClient
{
return DB::connection((new static())->connection)->getClient();
}
public function resolveConnection(): Connection
{
return DB::connection($this->connection);
}
}
tests.bootstrap.sh
+1
View File
@@ -15,6 +15,7 @@ cp /src/tests/config/app.php /app/config/app.php
cp /src/tests/migrations/exampleTable.php /app/database/migrations/2022_01_01_000000_example.php
cp /src/tests/migrations/example2Table.php /app/database/migrations/2022_01_01_000001_example.php
cp /src/tests/migrations/example3Table.php /app/database/migrations/2022_01_01_000002_example.php
cp /src/tests/migrations/example4Table.php /app/database/migrations/2022_01_01_000003_example.php
cat /src/tests/config/.env >> /app/.env
# Creating test tables
tests/ClusterTest.php
+47
View File
@@ -0,0 +1,47 @@
<?php
namespace Tests;
use Tests\Models\Example4;
use Tests\Models\Example4Problem;
class ClusterTest extends TestCase
{
protected function setUp(): void
{
parent::setUp();
Example4::truncate();
}
public function testRegularCluster()
{
Example4::truncate();
sleep(1); // clickhouse nodes sync lag
$this->assertEquals('clickhouse01', (new Example4())->getThisClient()->getConnectHost());
Example4::insertAssoc([['f_int' => 1, 'f_string' => 'a']]);
$this->assertNotEmpty(Example4::where('f_int', 1)->getRows());
(new Example4())->resolveConnection()->getCluster()->slideNode();
$this->assertEquals('clickhouse02', (new Example4())->getThisClient()->getConnectHost());
$this->assertEmpty(Example4::where('f_int', 1)->getRows());
sleep(1); // clickhouse nodes sync lag
$this->assertNotEmpty(Example4::where('f_int', 1)->getRows());
(new Example4())->resolveConnection()->getCluster()->slideNode();
$this->assertEquals('clickhouse01', (new Example4())->getThisClient()->getConnectHost());
}
public function testProblemCluster()
{
$this->assertEquals('clickhouse02', (new Example4Problem())->getThisClient()->getConnectHost());
Example4Problem::insertAssoc([['f_int' => 1, 'f_string' => 'a']]);
$this->assertNotEmpty(Example4Problem::where('f_int', 1)->getRows());
(new Example4Problem())->resolveConnection()->getCluster()->slideNode();
$this->assertEquals('clickhouse02', (new Example4Problem())->getThisClient()->getConnectHost());
(new Example4Problem())->resolveConnection()->getCluster()->slideNode();
$this->assertEquals('clickhouse02', (new Example4Problem())->getThisClient()->getConnectHost());
}
}
tests/Models/Example4.php
+15
View File
@@ -0,0 +1,15 @@
<?php
namespace Tests\Models;
use PhpClickHouseLaravel\BaseModel;
/**
* @property string $f_string
* @property int $f_int
*/
class Example4 extends BaseModel
{
protected $connection = 'clickhouse-cluster';
protected $table = 'examples4';
}
tests/Models/Example4Problem.php
+15
View File
@@ -0,0 +1,15 @@
<?php
namespace Tests\Models;
use PhpClickHouseLaravel\BaseModel;
/**
* @property string $f_string
* @property int $f_int
*/
class Example4Problem extends BaseModel
{
protected $connection = 'problem-clickhouse-cluster';
protected $table = 'examples4';
}
tests/config/.env
+1 -1
View File
@@ -1,5 +1,5 @@
CLICKHOUSE_HOST=clickhouse
CLICKHOUSE_HOST=clickhouse01
CLICKHOUSE_PORT=8123
CLICKHOUSE_DATABASE=default
CLICKHOUSE_USERNAME=default
tests/config/database.php
+43 -1
View File
@@ -53,7 +53,7 @@ return [
'clickhouse2' => [
'driver' => 'clickhouse',
'host' => 'clickhouse2',
'host' => 'clickhouse02',
'port' => '8123',
'database' => 'default',
'username' => 'default',
@@ -63,6 +63,48 @@ return [
'https' => false,
'retries' => 0,
],
'clickhouse-cluster' => [
'driver' => 'clickhouse',
'cluster' => [
[
'host' => 'clickhouse01',
'port' => '8123',
],
[
'host' => 'clickhouse02',
'port' => '8123',
],
],
'database' => 'default',
'username' => 'default',
'password' => '',
'timeout_connect' => 2,
'timeout_query' => 2,
'https' => false,
'retries' => 0,
],
'problem-clickhouse-cluster' => [
'driver' => 'clickhouse',
'cluster' => [
[
'host' => 'clickhouse03', // non-existent node
'port' => '8123',
],
[
'host' => 'clickhouse02',
'port' => '8123',
],
],
'database' => 'default',
'username' => 'default',
'password' => '',
'timeout_connect' => 2,
'timeout_query' => 2,
'https' => false,
'retries' => 0,
],
],
/*
tests/docker/clickhouse01/config.xml
+70
View File
@@ -0,0 +1,70 @@
<?xml version="1.0"?>
<company>
<logger>
<level>debug</level>
<console>true</console>
<log remove="remove"/>
<errorlog remove="remove"/>
</logger>
<query_log>
<database>system</database>
<table>query_log</table>
</query_log>
<listen_host>0.0.0.0</listen_host>
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<interserver_http_host>clickhouse01</interserver_http_host>
<interserver_http_port>9009</interserver_http_port>
<max_connections>4096</max_connections>
<keep_alive_timeout>3</keep_alive_timeout>
<max_concurrent_queries>100</max_concurrent_queries>
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
<path>/var/lib/clickhouse/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<users_config>users.xml</users_config>
<default_profile>default</default_profile>
<default_database>default</default_database>
<timezone>UTC</timezone>
<mlock_executable>false</mlock_executable>
<remote_servers>
<company_cluster>
<shard>
<replica>
<host>clickhouse01</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse02</host>
<port>9000</port>
</replica>
</shard>
</company_cluster>
</remote_servers>
<zookeeper>
<node index="1">
<host>zookeeper</host>
<port>2181</port>
</node>
</zookeeper>
<macros>
<cluster>company_cluster</cluster>
<shard>01</shard>
<replica>clickhouse01</replica>
</macros>
<distributed_ddl>
<path>/clickhouse/task_queue/ddl</path>
</distributed_ddl>
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
</company>
tests/docker/clickhouse01/users.xml
+43
View File
@@ -0,0 +1,43 @@
<?xml version="1.0"?>
<company>
<profiles>
<default>
<max_memory_usage>10000000000</max_memory_usage>
<use_uncompressed_cache>0</use_uncompressed_cache>
<load_balancing>in_order</load_balancing>
<log_queries>1</log_queries>
</default>
</profiles>
<users>
<default>
<password></password>
<profile>default</profile>
<networks>
<ip>::/0</ip>
</networks>
<quota>default</quota>
</default>
<admin>
<password>123</password>
<profile>default</profile>
<networks>
<ip>::/0</ip>
</networks>
<quota>default</quota>
</admin>
</users>
<quotas>
<default>
<interval>
<duration>3600</duration>
<queries>0</queries>
<errors>0</errors>
<result_rows>0</result_rows>
<read_rows>0</read_rows>
<execution_time>0</execution_time>
</interval>
</default>
</quotas>
</company>
tests/docker/clickhouse02/config.xml
+70
View File
@@ -0,0 +1,70 @@
<?xml version="1.0"?>
<company>
<logger>
<level>debug</level>
<console>true</console>
<log remove="remove"/>
<errorlog remove="remove"/>
</logger>
<query_log>
<database>system</database>
<table>query_log</table>
</query_log>
<listen_host>0.0.0.0</listen_host>
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<interserver_http_host>clickhouse02</interserver_http_host>
<interserver_http_port>9009</interserver_http_port>
<max_connections>4096</max_connections>
<keep_alive_timeout>3</keep_alive_timeout>
<max_concurrent_queries>100</max_concurrent_queries>
<uncompressed_cache_size>8589934592</uncompressed_cache_size>
<mark_cache_size>5368709120</mark_cache_size>
<path>/var/lib/clickhouse/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<users_config>users.xml</users_config>
<default_profile>default</default_profile>
<default_database>default</default_database>
<timezone>UTC</timezone>
<mlock_executable>false</mlock_executable>
<remote_servers>
<company_cluster>
<shard>
<replica>
<host>clickhouse01</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse02</host>
<port>9000</port>
</replica>
</shard>
</company_cluster>
</remote_servers>
<zookeeper>
<node index="1">
<host>zookeeper</host>
<port>2181</port>
</node>
</zookeeper>
<macros>
<cluster>company_cluster</cluster>
<shard>01</shard>
<replica>clickhouse02</replica>
</macros>
<distributed_ddl>
<path>/clickhouse/task_queue/ddl</path>
</distributed_ddl>
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
</company>
tests/docker/clickhouse02/users.xml
+43
View File
@@ -0,0 +1,43 @@
<?xml version="1.0"?>
<company>
<profiles>
<default>
<max_memory_usage>10000000000</max_memory_usage>
<use_uncompressed_cache>0</use_uncompressed_cache>
<load_balancing>in_order</load_balancing>
<log_queries>1</log_queries>
</default>
</profiles>
<users>
<default>
<password></password>
<profile>default</profile>
<networks>
<ip>::/0</ip>
</networks>
<quota>default</quota>
</default>
<admin>
<password>123</password>
<profile>default</profile>
<networks>
<ip>::/0</ip>
</networks>
<quota>default</quota>
</admin>
</users>
<quotas>
<default>
<interval>
<duration>3600</duration>
<queries>0</queries>
<errors>0</errors>
<result_rows>0</result_rows>
<read_rows>0</read_rows>
<execution_time>0</execution_time>
</interval>
</default>
</quotas>
</company>
tests/migrations/example4Table.php
+36
View File
@@ -0,0 +1,36 @@
<?php
return new class extends \PhpClickHouseLaravel\Migration {
protected $connection = 'clickhouse-cluster';
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
static::write(
"
CREATE TABLE IF NOT EXISTS examples4 (
created_at DateTime64 DEFAULT now64(),
f_int Int64,
f_string String
)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/default.examples4', '{replica}')
ORDER BY (f_int)
"
);
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
static::write('DROP TABLE examples4');
}
};