Ecoer Logo

@highland0971

25
steemit.com/@highland0971
VOTING POWER100.00%
DOWNVOTE POWER100.00%
RESOURCE CREDITS100.00%
REPUTATION PROGRESS0.00%
Net Worth
0.047USD
STEEM
0.000STEEM
SBD
0.021SBD
Effective Power
5.007SP
├── Own SP
0.629SP
└── Incoming Deleg
+4.378SP

Detailed Balance

STEEM
balance
0.000STEEM
market_balance
0.000STEEM
savings_balance
0.000STEEM
reward_steem_balance
0.000STEEM
STEEM POWER
Own SP
0.629SP
Delegated Out
0.000SP
Delegation In
4.378SP
Effective Power
5.007SP
Reward SP (pending)
0.007SP
SBD
sbd_balance
0.000SBD
sbd_conversions
0.000SBD
sbd_market_balance
0.000SBD
savings_sbd_balance
0.000SBD
reward_sbd_balance
0.021SBD
{
  "balance": "0.000 STEEM",
  "savings_balance": "0.000 STEEM",
  "reward_steem_balance": "0.000 STEEM",
  "vesting_shares": "1023.007539 VESTS",
  "delegated_vesting_shares": "0.000000 VESTS",
  "received_vesting_shares": "7120.652267 VESTS",
  "sbd_balance": "0.000 SBD",
  "savings_sbd_balance": "0.000 SBD",
  "reward_sbd_balance": "0.021 SBD",
  "conversions": []
}

Account Info

namehighland0971
id704820
rank605,039
reputation134634426
created2018-02-01T01:13:39
recovery_accountsteem
proxyNone
post_count6
comment_count0
lifetime_vote_count0
witnesses_voted_for0
last_post2018-02-11T15:37:00
last_root_post2018-02-11T15:37:00
last_vote_time2018-02-01T02:04:03
proxied_vsf_votes0, 0, 0, 0
can_vote1
voting_power0
delayed_votes0
balance0.000 STEEM
savings_balance0.000 STEEM
sbd_balance0.000 SBD
savings_sbd_balance0.000 SBD
vesting_shares1023.007539 VESTS
delegated_vesting_shares0.000000 VESTS
received_vesting_shares7120.652267 VESTS
reward_vesting_balance14.317069 VESTS
vesting_balance0.000 STEEM
vesting_withdraw_rate0.000000 VESTS
next_vesting_withdrawal1969-12-31T23:59:59
withdrawn0
to_withdraw0
withdraw_routes0
savings_withdraw_requests0
last_account_recovery1970-01-01T00:00:00
reset_accountnull
last_owner_update1970-01-01T00:00:00
last_account_update2018-02-02T00:02:36
minedNo
sbd_seconds0
sbd_last_interest_payment1970-01-01T00:00:00
savings_sbd_last_interest_payment1970-01-01T00:00:00
{
  "active": {
    "account_auths": [],
    "key_auths": [
      [
        "STM8V52JgqPnhC8Fo7tsoWPuagrV9T3afu2GFPnVYzTEbwzHxiEZ3",
        1
      ]
    ],
    "weight_threshold": 1
  },
  "balance": "0.000 STEEM",
  "can_vote": true,
  "comment_count": 0,
  "created": "2018-02-01T01:13:39",
  "curation_rewards": 0,
  "delegated_vesting_shares": "0.000000 VESTS",
  "downvote_manabar": {
    "current_mana": 2035914951,
    "last_update_time": 1779066489
  },
  "guest_bloggers": [],
  "id": 704820,
  "json_metadata": "{\"profile\":{\"name\":\"Highland\",\"website\":\"https://highland0971.github.io/\",\"profile_image\":\"https://upload.jianshu.io/users/upload.jianshu.io/users/upload_avatars/4032965/0d1af180631a?imageMogr2/auto-orient/strip|imageView2/1/w/300/h/300\"}}",
  "last_account_recovery": "1970-01-01T00:00:00",
  "last_account_update": "2018-02-02T00:02:36",
  "last_owner_update": "1970-01-01T00:00:00",
  "last_post": "2018-02-11T15:37:00",
  "last_root_post": "2018-02-11T15:37:00",
  "last_vote_time": "2018-02-01T02:04:03",
  "lifetime_vote_count": 0,
  "market_history": [],
  "memo_key": "STM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp",
  "mined": false,
  "name": "highland0971",
  "next_vesting_withdrawal": "1969-12-31T23:59:59",
  "other_history": [],
  "owner": {
    "account_auths": [],
    "key_auths": [
      [
        "STM79QY49asjRKnpXEZdapGfjXg18r2EPFsXRxzjoNRo82WNXQ3QE",
        1
      ]
    ],
    "weight_threshold": 1
  },
  "pending_claimed_accounts": 0,
  "post_bandwidth": 0,
  "post_count": 6,
  "post_history": [],
  "posting": {
    "account_auths": [],
    "key_auths": [
      [
        "STM5z3eJEV19juX6NNmUQqYgnebToE7LYk5VJKKgZnGPM8vExbEWD",
        1
      ]
    ],
    "weight_threshold": 1
  },
  "posting_json_metadata": "{\"profile\":{\"name\":\"Highland\",\"website\":\"https://highland0971.github.io/\",\"profile_image\":\"https://upload.jianshu.io/users/upload.jianshu.io/users/upload_avatars/4032965/0d1af180631a?imageMogr2/auto-orient/strip|imageView2/1/w/300/h/300\"}}",
  "posting_rewards": 13,
  "proxied_vsf_votes": [
    0,
    0,
    0,
    0
  ],
  "proxy": "",
  "received_vesting_shares": "7120.652267 VESTS",
  "recovery_account": "steem",
  "reputation": 134634426,
  "reset_account": "null",
  "reward_sbd_balance": "0.021 SBD",
  "reward_steem_balance": "0.000 STEEM",
  "reward_vesting_balance": "14.317069 VESTS",
  "reward_vesting_steem": "0.007 STEEM",
  "savings_balance": "0.000 STEEM",
  "savings_sbd_balance": "0.000 SBD",
  "savings_sbd_last_interest_payment": "1970-01-01T00:00:00",
  "savings_sbd_seconds": "0",
  "savings_sbd_seconds_last_update": "1970-01-01T00:00:00",
  "savings_withdraw_requests": 0,
  "sbd_balance": "0.000 SBD",
  "sbd_last_interest_payment": "1970-01-01T00:00:00",
  "sbd_seconds": "0",
  "sbd_seconds_last_update": "1970-01-01T00:00:00",
  "tags_usage": [],
  "to_withdraw": 0,
  "transfer_history": [],
  "vesting_balance": "0.000 STEEM",
  "vesting_shares": "1023.007539 VESTS",
  "vesting_withdraw_rate": "0.000000 VESTS",
  "vote_history": [],
  "voting_manabar": {
    "current_mana": "8143659806",
    "last_update_time": 1779066489
  },
  "voting_power": 0,
  "withdraw_routes": 0,
  "withdrawn": 0,
  "witness_votes": [],
  "witnesses_voted_for": 0,
  "rank": 605039
}

Withdraw Routes

IncomingOutgoing
Empty
Empty
{
  "incoming": [],
  "outgoing": []
}
From Date
To Date
steemdelegated 4.378 SP to @highland0971
2026/05/18 01:08:09
106,144,502|af38eca
delegateehighland0971
delegatorsteem
vesting shares7120.652267 VESTS
Transaction InfoBlock #106144502/Trx af38eca989b0a72a8ebe1f6b14700edb376e949e
View Raw JSON Data
{
  "block": 106144502,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "7120.652267 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2026-05-18T01:08:09",
  "trx_id": "af38eca989b0a72a8ebe1f6b14700edb376e949e",
  "trx_in_block": 2,
  "virtual_op": 0
}
steemdelegated 2.711 SP to @highland0971
2026/05/12 07:25:45
105,980,008|e56f699
delegateehighland0971
delegatorsteem
vesting shares4408.441862 VESTS
Transaction InfoBlock #105980008/Trx e56f699bac9da53d97d6d8464dbf42a975a02c76
View Raw JSON Data
{
  "block": 105980008,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "4408.441862 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2026-05-12T07:25:45",
  "trx_id": "e56f699bac9da53d97d6d8464dbf42a975a02c76",
  "trx_in_block": 3,
  "virtual_op": 0
}
steemdelegated 4.386 SP to @highland0971
2026/04/26 00:27:36
105,512,128|dff85bc
delegateehighland0971
delegatorsteem
vesting shares7133.168023 VESTS
Transaction InfoBlock #105512128/Trx dff85bc54807e001df4f2b28999cd122f514f89e
View Raw JSON Data
{
  "block": 105512128,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "7133.168023 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2026-04-26T00:27:36",
  "trx_id": "dff85bc54807e001df4f2b28999cd122f514f89e",
  "trx_in_block": 1,
  "virtual_op": 0
}
steemdelegated 2.736 SP to @highland0971
2026/01/23 10:10:21
102,854,653|03bf6fe
delegateehighland0971
delegatorsteem
vesting shares4449.988681 VESTS
Transaction InfoBlock #102854653/Trx 03bf6fed495ed705d244ce009aa68e3f2868f589
View Raw JSON Data
{
  "block": 102854653,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "4449.988681 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2026-01-23T10:10:21",
  "trx_id": "03bf6fed495ed705d244ce009aa68e3f2868f589",
  "trx_in_block": 2,
  "virtual_op": 0
}
steemdelegated 2.837 SP to @highland0971
2024/12/17 05:28:18
91,301,031|33b584c
delegateehighland0971
delegatorsteem
vesting shares4614.207878 VESTS
Transaction InfoBlock #91301031/Trx 33b584ce5d137cb94632171600c466e8d8d4ab51
View Raw JSON Data
{
  "block": 91301031,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "4614.207878 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2024-12-17T05:28:18",
  "trx_id": "33b584ce5d137cb94632171600c466e8d8d4ab51",
  "trx_in_block": 2,
  "virtual_op": 0
}
steemdelegated 2.941 SP to @highland0971
2023/11/13 21:10:42
79,855,222|aff3879
delegateehighland0971
delegatorsteem
vesting shares4783.341410 VESTS
Transaction InfoBlock #79855222/Trx aff38795db8bf0bfed2c06b417daedc7a1022ab7
View Raw JSON Data
{
  "block": 79855222,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "4783.341410 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2023-11-13T21:10:42",
  "trx_id": "aff38795db8bf0bfed2c06b417daedc7a1022ab7",
  "trx_in_block": 0,
  "virtual_op": 0
}
steemdelegated 4.747 SP to @highland0971
2023/09/21 22:52:15
78,349,074|fe36973
delegateehighland0971
delegatorsteem
vesting shares7720.620196 VESTS
Transaction InfoBlock #78349074/Trx fe36973e53e16b0fa522b1f9f74d4bc999de4374
View Raw JSON Data
{
  "block": 78349074,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "7720.620196 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2023-09-21T22:52:15",
  "trx_id": "fe36973e53e16b0fa522b1f9f74d4bc999de4374",
  "trx_in_block": 0,
  "virtual_op": 0
}
steemdelegated 4.883 SP to @highland0971
2022/11/03 12:31:51
69,114,250|4e373a5
delegateehighland0971
delegatorsteem
vesting shares7942.301634 VESTS
Transaction InfoBlock #69114250/Trx 4e373a56486d3672f8b848e39825b231383f7337
View Raw JSON Data
{
  "block": 69114250,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "7942.301634 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2022-11-03T12:31:51",
  "trx_id": "4e373a56486d3672f8b848e39825b231383f7337",
  "trx_in_block": 0,
  "virtual_op": 0
}
steemdelegated 5.019 SP to @highland0971
2022/01/17 11:43:39
60,810,335|2f941e0
delegateehighland0971
delegatorsteem
vesting shares8162.834865 VESTS
Transaction InfoBlock #60810335/Trx 2f941e06065c07a7f96bd51e7fdd4f6cbc2befa6
View Raw JSON Data
{
  "block": 60810335,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "8162.834865 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2022-01-17T11:43:39",
  "trx_id": "2f941e06065c07a7f96bd51e7fdd4f6cbc2befa6",
  "trx_in_block": 4,
  "virtual_op": 0
}
steemdelegated 5.132 SP to @highland0971
2021/06/14 01:36:42
54,608,676|cedcc4f
delegateehighland0971
delegatorsteem
vesting shares8346.603523 VESTS
Transaction InfoBlock #54608676/Trx cedcc4f832252066d8fd3380f86220bc3c785f9a
View Raw JSON Data
{
  "block": 54608676,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "8346.603523 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2021-06-14T01:36:42",
  "trx_id": "cedcc4f832252066d8fd3380f86220bc3c785f9a",
  "trx_in_block": 4,
  "virtual_op": 0
}
steemdelegated 5.247 SP to @highland0971
2020/12/11 11:54:12
49,356,096|2b52846
delegateehighland0971
delegatorsteem
vesting shares8534.025497 VESTS
Transaction InfoBlock #49356096/Trx 2b52846b5fa3141ff38a9f91d6fada3c9d85a1cc
View Raw JSON Data
{
  "block": 49356096,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "8534.025497 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2020-12-11T11:54:12",
  "trx_id": "2b52846b5fa3141ff38a9f91d6fada3c9d85a1cc",
  "trx_in_block": 4,
  "virtual_op": 0
}
steemdelegated 1.176 SP to @highland0971
2020/12/06 05:31:21
49,207,659|f5ca665
delegateehighland0971
delegatorsteem
vesting shares1912.543513 VESTS
Transaction InfoBlock #49207659/Trx f5ca6655184b29547ee5e0f83f3462cda8a6ba9c
View Raw JSON Data
{
  "block": 49207659,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "1912.543513 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2020-12-06T05:31:21",
  "trx_id": "f5ca6655184b29547ee5e0f83f3462cda8a6ba9c",
  "trx_in_block": 3,
  "virtual_op": 0
}
steemdelegated 5.251 SP to @highland0971
2020/12/05 15:32:12
49,191,193|6405f46
delegateehighland0971
delegatorsteem
vesting shares8540.233351 VESTS
Transaction InfoBlock #49191193/Trx 6405f4683dddbb77df21424f0b70956f514541fa
View Raw JSON Data
{
  "block": 49191193,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "8540.233351 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2020-12-05T15:32:12",
  "trx_id": "6405f4683dddbb77df21424f0b70956f514541fa",
  "trx_in_block": 0,
  "virtual_op": 0
}
steemdelegated 1.181 SP to @highland0971
2020/11/02 17:14:18
48,259,685|7b3dbb9
delegateehighland0971
delegatorsteem
vesting shares1920.017158 VESTS
Transaction InfoBlock #48259685/Trx 7b3dbb94a79d3927d08cce137a28da7c7abad872
View Raw JSON Data
{
  "block": 48259685,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "1920.017158 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2020-11-02T17:14:18",
  "trx_id": "7b3dbb94a79d3927d08cce137a28da7c7abad872",
  "trx_in_block": 2,
  "virtual_op": 0
}
steemdelegated 5.376 SP to @highland0971
2020/05/09 06:29:27
43,217,918|c84871a
delegateehighland0971
delegatorsteem
vesting shares8743.038710 VESTS
Transaction InfoBlock #43217918/Trx c84871ad63b7be8b3b272eb3fd1a21bb1536c188
View Raw JSON Data
{
  "block": 43217918,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "8743.038710 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2020-05-09T06:29:27",
  "trx_id": "c84871ad63b7be8b3b272eb3fd1a21bb1536c188",
  "trx_in_block": 2,
  "virtual_op": 0
}
steemdelegated 1.201 SP to @highland0971
2020/05/08 10:12:48
43,194,153|7c14bee
delegateehighland0971
delegatorsteem
vesting shares1953.311140 VESTS
Transaction InfoBlock #43194153/Trx 7c14bee40ea3adefd4e1baef429a9dced9b1f430
View Raw JSON Data
{
  "block": 43194153,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "1953.311140 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2020-05-08T10:12:48",
  "trx_id": "7c14bee40ea3adefd4e1baef429a9dced9b1f430",
  "trx_in_block": 5,
  "virtual_op": 0
}
steemdelegated 5.384 SP to @highland0971
2020/04/16 00:23:57
42,565,792|9caa9d5
delegateehighland0971
delegatorsteem
vesting shares8755.926158 VESTS
Transaction InfoBlock #42565792/Trx 9caa9d572e20f9075bba638c7542adede6f04812
View Raw JSON Data
{
  "block": 42565792,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "8755.926158 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2020-04-16T00:23:57",
  "trx_id": "9caa9d572e20f9075bba638c7542adede6f04812",
  "trx_in_block": 6,
  "virtual_op": 0
}
steemitboardreplied to @highland0971 / steemitboard-notify-highland0971-20200201t014756000z
2020/02/01 01:47:57
40,425,541|641abbe
authorsteemitboard
bodyCongratulations @highland0971! You received a personal award! <table><tr><td>https://steemitimages.com/70x70/http://steemitboard.com/@highland0971/birthday2.png</td><td>Happy Birthday! - You are on the Steem blockchain for 2 years!</td></tr></table> <sub>_You can view [your badges on your Steem Board](https://steemitboard.com/@highland0971) and compare to others on the [Steem Ranking](https://steemitboard.com/ranking/index.php?name=highland0971)_</sub> ###### [Vote for @Steemitboard as a witness](https://v2.steemconnect.com/sign/account-witness-vote?witness=steemitboard&approve=1) to get one more award and increased upvotes!
json metadata{"image":["https://steemitboard.com/img/notify.png"]}
parent authorhighland0971
parent permlinkhow-to-compile-eos-on-clean-centos-system-updated
permlinksteemitboard-notify-highland0971-20200201t014756000z
title
Transaction InfoBlock #40425541/Trx 641abbe7d1f4ff483e81488ea2ae46ff04d9590e
View Raw JSON Data
{
  "block": 40425541,
  "op": [
    "comment",
    {
      "author": "steemitboard",
      "body": "Congratulations @highland0971! You received a personal award!\n\n<table><tr><td>https://steemitimages.com/70x70/http://steemitboard.com/@highland0971/birthday2.png</td><td>Happy Birthday! - You are on the Steem blockchain for 2 years!</td></tr></table>\n\n<sub>_You can view [your badges on your Steem Board](https://steemitboard.com/@highland0971) and compare to others on the [Steem Ranking](https://steemitboard.com/ranking/index.php?name=highland0971)_</sub>\n\n\n###### [Vote for @Steemitboard as a witness](https://v2.steemconnect.com/sign/account-witness-vote?witness=steemitboard&approve=1) to get one more award and increased upvotes!",
      "json_metadata": "{\"image\":[\"https://steemitboard.com/img/notify.png\"]}",
      "parent_author": "highland0971",
      "parent_permlink": "how-to-compile-eos-on-clean-centos-system-updated",
      "permlink": "steemitboard-notify-highland0971-20200201t014756000z",
      "title": ""
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2020-02-01T01:47:57",
  "trx_id": "641abbe7d1f4ff483e81488ea2ae46ff04d9590e",
  "trx_in_block": 2,
  "virtual_op": 0
}
steemdelegated 5.504 SP to @highland0971
2019/05/12 17:28:57
32,848,582|2b3ff54
delegateehighland0971
delegatorsteem
vesting shares8951.542971 VESTS
Transaction InfoBlock #32848582/Trx 2b3ff54f68e4b9e66fc963296b6eb209aa6619b4
View Raw JSON Data
{
  "block": 32848582,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "8951.542971 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2019-05-12T17:28:57",
  "trx_id": "2b3ff54f68e4b9e66fc963296b6eb209aa6619b4",
  "trx_in_block": 27,
  "virtual_op": 0
}
steemitboardreplied to @highland0971 / steemitboard-notify-highland0971-20190201t021153000z
2019/02/01 02:11:54
29,953,746|cb9c1e0
authorsteemitboard
bodyCongratulations @highland0971! You received a personal award! <table><tr><td>https://steemitimages.com/70x70/http://steemitboard.com/@highland0971/birthday1.png</td><td>Happy Birthday! - You are on the Steem blockchain for 1 year!</td></tr></table> <sub>_[Click here to view your Board](https://steemitboard.com/@highland0971)_</sub> > Support [SteemitBoard's project](https://steemit.com/@steemitboard)! **[Vote for its witness](https://v2.steemconnect.com/sign/account-witness-vote?witness=steemitboard&approve=1)** and **get one more award**!
json metadata{"image":["https://steemitboard.com/img/notify.png"]}
parent authorhighland0971
parent permlinkhow-to-compile-eos-on-clean-centos-system-updated
permlinksteemitboard-notify-highland0971-20190201t021153000z
title
Transaction InfoBlock #29953746/Trx cb9c1e0f3f6c399c88cda9152ca24a3ec3b36877
View Raw JSON Data
{
  "block": 29953746,
  "op": [
    "comment",
    {
      "author": "steemitboard",
      "body": "Congratulations @highland0971! You received a personal award!\n\n<table><tr><td>https://steemitimages.com/70x70/http://steemitboard.com/@highland0971/birthday1.png</td><td>Happy Birthday! - You are on the Steem blockchain for 1 year!</td></tr></table>\n\n<sub>_[Click here to view your Board](https://steemitboard.com/@highland0971)_</sub>\n\n\n> Support [SteemitBoard's project](https://steemit.com/@steemitboard)! **[Vote for its witness](https://v2.steemconnect.com/sign/account-witness-vote?witness=steemitboard&approve=1)** and **get one more award**!",
      "json_metadata": "{\"image\":[\"https://steemitboard.com/img/notify.png\"]}",
      "parent_author": "highland0971",
      "parent_permlink": "how-to-compile-eos-on-clean-centos-system-updated",
      "permlink": "steemitboard-notify-highland0971-20190201t021153000z",
      "title": ""
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2019-02-01T02:11:54",
  "trx_id": "cb9c1e0f3f6c399c88cda9152ca24a3ec3b36877",
  "trx_in_block": 1,
  "virtual_op": 0
}
steemdelegated 5.627 SP to @highland0971
2018/05/16 20:20:27
22,489,910|40b5f7c
delegateehighland0971
delegatorsteem
vesting shares9151.191369 VESTS
Transaction InfoBlock #22489910/Trx 40b5f7ca03f18533e9ca741900e797b8a4e368e6
View Raw JSON Data
{
  "block": 22489910,
  "op": [
    "delegate_vesting_shares",
    {
      "delegatee": "highland0971",
      "delegator": "steem",
      "vesting_shares": "9151.191369 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-05-16T20:20:27",
  "trx_id": "40b5f7ca03f18533e9ca741900e797b8a4e368e6",
  "trx_in_block": 8,
  "virtual_op": 0
}
teslamaxwellupvoted (100.00%) @highland0971 / how-to-compile-eos-on-clean-centos-system
2018/03/19 21:55:57
20,823,201|1be913f
authorhighland0971
permlinkhow-to-compile-eos-on-clean-centos-system
voterteslamaxwell
weight10000 (100.00%)
Transaction InfoBlock #20823201/Trx 1be913f84c10b6d9fbceac55176e026e2ab55919
View Raw JSON Data
{
  "block": 20823201,
  "op": [
    "vote",
    {
      "author": "highland0971",
      "permlink": "how-to-compile-eos-on-clean-centos-system",
      "voter": "teslamaxwell",
      "weight": 10000
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-03-19T21:55:57",
  "trx_id": "1be913f84c10b6d9fbceac55176e026e2ab55919",
  "trx_in_block": 80,
  "virtual_op": 0
}
sunnysunreplied to @highland0971 / re-highland0971-how-to-compile-eos-on-clean-centos-system-20180212t092608048z
2018/02/12 09:26:39
19,801,140|5999bf9
authorsunnysun
body@@ -46,16 +46,17 @@ ommand: +%60 /root/tm @@ -213,16 +213,17 @@ tatement +%60 %0A%0AAny id
json metadata{"tags":["eos"],"app":"steemit/0.1"}
parent authorhighland0971
parent permlinkhow-to-compile-eos-on-clean-centos-system
permlinkre-highland0971-how-to-compile-eos-on-clean-centos-system-20180212t092608048z
title
Transaction InfoBlock #19801140/Trx 5999bf93c55660321d6f61ec25f97c8f7e7961d1
View Raw JSON Data
{
  "block": 19801140,
  "op": [
    "comment",
    {
      "author": "sunnysun",
      "body": "@@ -46,16 +46,17 @@\n ommand: \n+%60\n /root/tm\n@@ -213,16 +213,17 @@\n tatement\n+%60\n %0A%0AAny id\n",
      "json_metadata": "{\"tags\":[\"eos\"],\"app\":\"steemit/0.1\"}",
      "parent_author": "highland0971",
      "parent_permlink": "how-to-compile-eos-on-clean-centos-system",
      "permlink": "re-highland0971-how-to-compile-eos-on-clean-centos-system-20180212t092608048z",
      "title": ""
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-12T09:26:39",
  "trx_id": "5999bf93c55660321d6f61ec25f97c8f7e7961d1",
  "trx_in_block": 17,
  "virtual_op": 0
}
sunnysunreplied to @highland0971 / re-highland0971-how-to-compile-eos-on-clean-centos-system-20180212t092608048z
2018/02/12 09:26:12
19,801,131|64d26a3
authorsunnysun
bodyPart 4, always got errors to run `make -j4` command: /root/tmp/eos/libraries/fc/include/fc/utility.hpp:63:3: error: body of constexpr function ‘constexpr std::size_t fc::const_strlen(const char*)’ not a return-statement Any ideas? my gcc version is 4.8.5, seems I have to upgrade gcc with higher version, right?
json metadata{"tags":["eos"],"app":"steemit/0.1"}
parent authorhighland0971
parent permlinkhow-to-compile-eos-on-clean-centos-system
permlinkre-highland0971-how-to-compile-eos-on-clean-centos-system-20180212t092608048z
title
Transaction InfoBlock #19801131/Trx 64d26a307253804f90ea4b8db88a096e2cad206f
View Raw JSON Data
{
  "block": 19801131,
  "op": [
    "comment",
    {
      "author": "sunnysun",
      "body": "Part 4, always got errors to run `make -j4` command: /root/tmp/eos/libraries/fc/include/fc/utility.hpp:63:3: error: body of constexpr function ‘constexpr std::size_t fc::const_strlen(const char*)’ not a return-statement\n\nAny ideas? my gcc version is 4.8.5, seems I have to upgrade gcc with higher version, right?",
      "json_metadata": "{\"tags\":[\"eos\"],\"app\":\"steemit/0.1\"}",
      "parent_author": "highland0971",
      "parent_permlink": "how-to-compile-eos-on-clean-centos-system",
      "permlink": "re-highland0971-how-to-compile-eos-on-clean-centos-system-20180212t092608048z",
      "title": ""
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-12T09:26:12",
  "trx_id": "64d26a307253804f90ea4b8db88a096e2cad206f",
  "trx_in_block": 14,
  "virtual_op": 0
}
highland0971published a new post: how-to-compile-eos-on-clean-centos-system-updated
2018/02/11 15:37:00
19,779,764|720ea4c
authorhighland0971
bodyPreviously, I've posted an [article](https://steemit.com/eos/@highland0971/how-to-compile-eos-on-clean-centos-system) about initial compilation of EOS in CentOS, missing mangodb driver and plugin. Now it's completed in this updated tutorial. There is a lot of posts about how to compile EOS on MacOS and Ubuntu, and the ***[official site](https://github.com/EOSIO/eos/wiki/Local-Environment#22-manual-build-script)*** has given a complete compile procedure guidence & automatic scripts to help the developers. However the choice of operting system is subjected to the actual cluster we use in production environment, and the official did not give a clue how to compile on different systems. According to my own compile practice, there are a lot of bugs and traps that need to be avoid when you compile the EOS. To help the developers who are anxious to experience the EOS on their prefered operating system, I write the following article to demonstrate how I compile on CentOS 7 and how to avoid the traps that hidden under the ***[offical guide](https://github.com/EOSIO/eos/wiki/Local-Environment#2-building-eos)*** . # 1. Hardware and System requirements Due to the requirements of building LLVM WebAssembly，the host must have 10GB free space and the RAM need to be greater than 16GB, multiple cpu cores is suggested. The operating system we use is CentOS 7.4 , you may choose other Redhat releases. # 2. Basic compile environment install According to the description on EOS.IO github official site,the EOS project is composed on C++14 stander, the default compile come with CentOS 7 is GNU 4.8.5, thus we need to manual choose different repos to install higher verison of GNU compile toolchain. Besides, the CMake version in CentOS 7 official site is a bit old, we need to install the newest Cmake manually. - Cmake 3.10.2 - GNU C/C++ compiler 6 serialies ```bash export BUILD_TEMP=${HOME}/tmp mkdir ${BUILD_TEMP} #GNU C++ 6.3.1 sudo yum install -y centos-release-scl sudo yum install -y devtoolset-6 scl enable devtoolset-6 bash #Ninja build tools sudo yum install -y epel-release && \ sudo yum install -y ninja-build #CMake 3.10.2 export BUILD_TEMP=${HOME}/tmp mkdir ${BUILD_TEMP} cd ${BUILD_TEMP} mkdir ${HOME}/opt curl -L https://cmake.org/files/v3.10/cmake-3.10.2-Linux-x86_64.tar.gz -o cmake-3.10.2-Linux-x86_64.tar.gz tar -zxf cmake-3.10.2-Linux-x86_64.tar.gz -C ${HOME}/opt export PATH=$PATH:${HOME}/opt/cmake-3.10.2-Linux-x86_64/bin/ #Other compile tool chain sudo yum install -y git autoconf automake libtool doxygen ocaml gmp-devel python-devel bzip2-devel openssl-devel libicu-devel bzip2 ``` # 3. Install EOS.IO dependencies - Boost 1.64 - secp256k1-zkp - Binaryen - LLVM 4.0 with WebAssembly (On actual compile environment, you also have to compile X86 target for a clean CentOS installation) ## 3.1 Compile Boost ```bash export BOOST_ROOT=${HOME}/opt/boost_1_64_0 cd ${BUILD_TEMP} curl -L https://sourceforge.net/projects/boost/files/boost/1.64.0/boost_1_64_0.tar.bz2 > boost_1.64.0.tar.bz2 && \ tar xvf boost_1.64.0.tar.bz2 && \ cd boost_1_64_0/ && \ ./bootstrap.sh "--prefix=$BOOST_ROOT" && \ ./b2 install ``` ## 3.2 Compile secp256k1-zkp ```bash cd ${BUILD_TEMP} git clone https://github.com/cryptonomex/secp256k1-zkp.git && \ cd secp256k1-zkp && \ ./autogen.sh && \ ./configure && \ make sudo make install ``` ## 3.3 Compile Binaryen ```bash cd ${BUILD_TEMP} git clone https://github.com/WebAssembly/binaryen && \ cd binaryen && \ git checkout tags/1.37.14 && \ cmake . && \ make -j 4 sudo make install ``` ## 3.4 Compile MangoDB driver ```bash cd ${BUILD_TEMP} wget https://github.com/mongodb/libbson/releases/download/1.9.2/libbson-1.9.2.tar.gz && \ tar -xzf libbson-1.9.2.tar.gz && \ cd libbson-1.9.2/ && \ ./configure && \ make -j4 sudo make install curl -L https://github.com/mongodb/mongo-c-driver/releases/download/1.8.0/mongo-c-driver-1.8.0.tar.gz -o mongo-c-driver-1.8.0.tar.gz && \ tar -xzf mongo-c-driver-1.8.0.tar.gz && \ cd mongo-c-driver-1.8.0 && \ ./configure --disable-automatic-init-and-cleanup && \ make -j4 sudo make install export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig git clone --depth 1 -b releases/stable git://github.com/mongodb/mongo-cxx-driver &&\ cd mongo-cxx-driver &&\ cmake -H. -Bbuild -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local && \ sudo cmake --build build --target install sudo vi /etc/ld.so.conf #Add following to files #/usr/local/lib sudo ldconfig ``` ## 3.5 Compile LLVM WebAssembly+ Host Target - TRAP 1 Due the EOS need LLVM to compile C++ to WebAssembly, the frontend of Clang and WebAssembly are both required, the official compile guidance only compile the WebAssembly target (I assume that is based on the clang is already installed on the ubuntu system,so the auther omitted it when compile LLVM for WebAssembly.), when we compile the LLVM wtih GNU compiler we need to both compile the WebAssembly and X86 target (which is achived by adding parameter `DLLVM_TARGETS_TO_BUILD='host'` to the cmake configuration), otherwise we would encounter `missing LLVMX86****.so` error when you compile EOS. - TRAP 2 At current, the EOS only support LLVM of version 4.0,which is achived by clone branch 40 from [llvm mirror site](https://github.com/llvm-mirror/llvm.git), **DO NOT** pull the latest source from github directlly without branch selection. - TRAP 3 You have to enable RTTI when you compile, otherwise you may encounter error like ***typeof ** undefine*** when you compile EOS. You can avoid this error by add parameter DLLVM_ENABLE_RTTI=ON to the cmake configuration. ```bash mkdir ${BUILD_TEMP}/wasm-compiler/build -p cd ${BUILD_TEMP}/wasm-compiler && \ git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git && \ cd llvm/tools && \ git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/clang.git && \ cd ${BUILD_TEMP}/wasm-compiler/build && \ cmake -G "Unix Makefiles" -DLLVM_ENABLE_RTTI=ON \ -DLLVM_TARGETS_TO_BUILD='host' \ -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly \ -DCMAKE_BUILD_TYPE=Release \ ../llvm sudo make -j4 install # export WASM_LLVM_CONFIG=/usr/local/bin/llvm-config ``` ## Compile EOS.IO - Tips: In default CMakefile, the mangodb plugin is disabled, to enable mangodb plugin, you have to add `-DBUILD_MONGO_DB_PLUGIN=ON` in configuration. ```bash cd git clone https://github.com/eosio/eos --recursive cd eos #If you need dawn 2.x version,uncomment following #git checkout dawn-2.x cmake -H. -B"build" -GNinja -DCMAKE_BUILD_TYPE=Release \ -DWASM_LLVM_CONFIG=/usr/local/bin/llvm-config \ -DCMAKE_INSTALL_PREFIX=/opt/eos \ -DBOOST_INCLUDEDIR=${BOOST_ROOT}/include/boost \ -DBUILD_MONGO_DB_PLUGIN=ON \ -DBOOST_LIBRARYDIR=${BOOST_ROOT}/lib sudo cmake --build build --target install ``` Now, the binaries are installed in build/install/bin directory ```bash ll build/install/bin/ # total 141276 # drwxrwxr-x. 2 highland highland 24 Feb 11 23:19 data-dir # -rwxr-xr-x. 1 highland highland 3056136 Feb 11 23:07 embed_genesis # -rwxr-xr-x. 1 highland highland 55330872 Feb 11 23:09 eosio-abigen # -rwxr-xr-x. 1 highland highland 4309440 Feb 11 23:08 eosioc # -rwxr-xr-x. 1 highland highland 35388792 Feb 11 23:08 eosio-codegen # -r-xr-xr-x. 1 highland highland 4195 Feb 11 22:57 eosiocpp # -rwxr-xr-x. 1 highland highland 39722544 Feb 11 23:08 eosiod # -rwxr-xr-x. 1 highland highland 2293128 Feb 11 23:08 eosio-launcher # -rwxr-xr-x. 1 highland highland 4545360 Feb 11 23:08 eosiowd ``` ## Start your first node Please refer to [official documents](https://github.com/EOSIO/eos/wiki/Local-Environment#4-creating-and-launching-a-single-node-testnet).
json metadata{"tags":["eos","centos","compile"],"links":["https://steemit.com/eos/@highland0971/how-to-compile-eos-on-clean-centos-system","https://github.com/EOSIO/eos/wiki/Local-Environment#22-manual-build-script","https://github.com/EOSIO/eos/wiki/Local-Environment#2-building-eos","https://github.com/llvm-mirror/llvm.git","https://github.com/EOSIO/eos/wiki/Local-Environment#4-creating-and-launching-a-single-node-testnet"],"app":"steemit/0.1","format":"markdown"}
parent author
parent permlinkeos
permlinkhow-to-compile-eos-on-clean-centos-system-updated
titleHow to compile EOS on clean CentOS system [updated]
Transaction InfoBlock #19779764/Trx 720ea4c5d3062697e53a0dccbfb5222f26f8a868
View Raw JSON Data
{
  "block": 19779764,
  "op": [
    "comment",
    {
      "author": "highland0971",
      "body": "Previously, I've posted an [article](https://steemit.com/eos/@highland0971/how-to-compile-eos-on-clean-centos-system) about initial compilation of EOS in CentOS, missing mangodb driver and plugin. Now it's completed in this updated tutorial.\n\nThere is a lot of posts about how to compile EOS on MacOS and Ubuntu, and the ***[official site](https://github.com/EOSIO/eos/wiki/Local-Environment#22-manual-build-script)*** has given a complete compile procedure guidence & automatic scripts to help the developers.\nHowever the choice of operting system is subjected to the actual cluster we use in production environment, and the official did not give a clue how to compile on different systems. According to my own compile practice, there are a lot of bugs and traps that need to be avoid when you compile the EOS. To help the developers who are anxious to experience the EOS on their prefered operating system, I write the following article to demonstrate how I compile on CentOS 7 and how to avoid the traps that hidden under the ***[offical guide](https://github.com/EOSIO/eos/wiki/Local-Environment#2-building-eos)*** .\n\n\n# 1. Hardware and System requirements\nDue to the requirements of building LLVM WebAssembly，the host must have 10GB free space and the RAM need to be greater than 16GB, multiple cpu cores is suggested.\n\nThe operating system we use is CentOS 7.4 , you may choose other Redhat releases.\n\n# 2. Basic compile environment install\n\nAccording to the description on EOS.IO github official site,the EOS project is composed on C++14 stander, the default compile come with CentOS 7 is GNU 4.8.5, thus we need to manual choose different repos to install higher verison of GNU compile toolchain. Besides, the CMake version in CentOS 7 official site is a bit old, we need to install the newest Cmake manually.\n\n- Cmake 3.10.2\n- GNU C/C++ compiler 6 serialies\n\n```bash\nexport BUILD_TEMP=${HOME}/tmp\nmkdir ${BUILD_TEMP}\n\n#GNU C++ 6.3.1\nsudo yum install -y centos-release-scl\nsudo yum install -y devtoolset-6\nscl enable devtoolset-6 bash\n\n#Ninja build tools\nsudo yum install -y epel-release && \\\nsudo yum install -y ninja-build\n\n#CMake 3.10.2\nexport BUILD_TEMP=${HOME}/tmp\nmkdir ${BUILD_TEMP}\n\ncd  ${BUILD_TEMP}\nmkdir ${HOME}/opt\ncurl -L https://cmake.org/files/v3.10/cmake-3.10.2-Linux-x86_64.tar.gz -o cmake-3.10.2-Linux-x86_64.tar.gz\ntar -zxf cmake-3.10.2-Linux-x86_64.tar.gz -C ${HOME}/opt\n\nexport PATH=$PATH:${HOME}/opt/cmake-3.10.2-Linux-x86_64/bin/\n\n#Other compile tool chain\nsudo yum install -y git autoconf automake libtool doxygen ocaml  gmp-devel python-devel bzip2-devel openssl-devel libicu-devel bzip2\n\n```\n\n# 3. Install EOS.IO dependencies\n- Boost 1.64\n- secp256k1-zkp\n- Binaryen\n- LLVM 4.0 with WebAssembly (On actual compile environment, you also have to compile X86 target for a clean CentOS installation)\n\n## 3.1 Compile Boost\n```bash\nexport BOOST_ROOT=${HOME}/opt/boost_1_64_0\ncd  ${BUILD_TEMP}\ncurl -L https://sourceforge.net/projects/boost/files/boost/1.64.0/boost_1_64_0.tar.bz2 > boost_1.64.0.tar.bz2 && \\\ntar xvf boost_1.64.0.tar.bz2 && \\\ncd boost_1_64_0/ && \\\n./bootstrap.sh \"--prefix=$BOOST_ROOT\" && \\\n./b2 install\n```\n\n## 3.2 Compile secp256k1-zkp\n```bash\ncd  ${BUILD_TEMP}\ngit clone https://github.com/cryptonomex/secp256k1-zkp.git && \\\ncd secp256k1-zkp && \\\n./autogen.sh && \\\n./configure && \\\nmake\nsudo make install\n```\n\n## 3.3 Compile Binaryen\n```bash\ncd  ${BUILD_TEMP}\ngit clone https://github.com/WebAssembly/binaryen && \\\ncd binaryen && \\\ngit checkout tags/1.37.14 && \\\ncmake . && \\\nmake -j 4\nsudo make install\n```\n\n## 3.4 Compile MangoDB driver\n```bash\ncd  ${BUILD_TEMP}\n\nwget https://github.com/mongodb/libbson/releases/download/1.9.2/libbson-1.9.2.tar.gz && \\\ntar -xzf libbson-1.9.2.tar.gz && \\\ncd libbson-1.9.2/ && \\\n./configure && \\\nmake -j4\nsudo make install\n\ncurl -L https://github.com/mongodb/mongo-c-driver/releases/download/1.8.0/mongo-c-driver-1.8.0.tar.gz -o mongo-c-driver-1.8.0.tar.gz && \\\ntar -xzf mongo-c-driver-1.8.0.tar.gz && \\\ncd mongo-c-driver-1.8.0 && \\\n./configure --disable-automatic-init-and-cleanup && \\\nmake -j4\nsudo make install\n\nexport PKG_CONFIG_PATH=/usr/local/lib/pkgconfig\n\ngit clone --depth 1 -b releases/stable git://github.com/mongodb/mongo-cxx-driver &&\\\ncd mongo-cxx-driver &&\\\ncmake -H. -Bbuild -G Ninja -DCMAKE_BUILD_TYPE=Release  -DCMAKE_INSTALL_PREFIX=/usr/local && \\\nsudo cmake --build build --target install\n\nsudo vi /etc/ld.so.conf\n#Add following to files\n#/usr/local/lib\nsudo ldconfig\n```\n\n## 3.5 Compile LLVM WebAssembly+ Host Target\n- TRAP 1\nDue the EOS need LLVM to compile C++ to WebAssembly, the frontend of Clang and WebAssembly are both required, the official compile guidance only compile the WebAssembly target (I assume that is based on the clang is already installed on the ubuntu system,so the auther omitted it when compile LLVM for WebAssembly.), when we compile the LLVM wtih GNU compiler we need to both compile the WebAssembly and X86 target (which is achived by adding parameter `DLLVM_TARGETS_TO_BUILD='host'` to the cmake configuration), otherwise we would encounter `missing LLVMX86****.so` error when you compile EOS.\n- TRAP 2\nAt current, the EOS only support LLVM of version 4.0,which is achived by clone branch 40 from [llvm mirror site](https://github.com/llvm-mirror/llvm.git), **DO NOT** pull the latest source from github directlly without branch selection.\n- TRAP 3\nYou have to enable RTTI when you compile, otherwise you may encounter error like ***typeof ** undefine*** when you compile EOS. You can avoid this error by add parameter DLLVM_ENABLE_RTTI=ON to the cmake configuration.\n\n```bash\n\nmkdir ${BUILD_TEMP}/wasm-compiler/build -p\ncd ${BUILD_TEMP}/wasm-compiler && \\\ngit clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git && \\\ncd llvm/tools && \\\ngit clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/clang.git && \\\ncd ${BUILD_TEMP}/wasm-compiler/build && \\\ncmake -G \"Unix Makefiles\" -DLLVM_ENABLE_RTTI=ON \\\n-DLLVM_TARGETS_TO_BUILD='host' \\\n-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly \\\n-DCMAKE_BUILD_TYPE=Release \\\n../llvm\n\nsudo make -j4 install\n\n# export WASM_LLVM_CONFIG=/usr/local/bin/llvm-config\n```\n\n## Compile EOS.IO\n\n- Tips: In default CMakefile, the mangodb plugin is disabled, to enable mangodb plugin, you have to add `-DBUILD_MONGO_DB_PLUGIN=ON` in configuration.\n\n```bash\ncd\ngit clone https://github.com/eosio/eos --recursive\ncd eos\n\n#If you need dawn 2.x version,uncomment following\n#git checkout dawn-2.x\n\ncmake -H. -B\"build\" -GNinja -DCMAKE_BUILD_TYPE=Release \\\n-DWASM_LLVM_CONFIG=/usr/local/bin/llvm-config \\\n-DCMAKE_INSTALL_PREFIX=/opt/eos \\\n-DBOOST_INCLUDEDIR=${BOOST_ROOT}/include/boost \\\n-DBUILD_MONGO_DB_PLUGIN=ON \\\n-DBOOST_LIBRARYDIR=${BOOST_ROOT}/lib\nsudo cmake --build build --target install\n```\nNow, the binaries are installed in build/install/bin directory\n```bash\nll build/install/bin/\n# total 141276\n# drwxrwxr-x. 2 highland highland       24 Feb 11 23:19 data-dir\n# -rwxr-xr-x. 1 highland highland  3056136 Feb 11 23:07 embed_genesis\n# -rwxr-xr-x. 1 highland highland 55330872 Feb 11 23:09 eosio-abigen\n# -rwxr-xr-x. 1 highland highland  4309440 Feb 11 23:08 eosioc\n# -rwxr-xr-x. 1 highland highland 35388792 Feb 11 23:08 eosio-codegen\n# -r-xr-xr-x. 1 highland highland     4195 Feb 11 22:57 eosiocpp\n# -rwxr-xr-x. 1 highland highland 39722544 Feb 11 23:08 eosiod\n# -rwxr-xr-x. 1 highland highland  2293128 Feb 11 23:08 eosio-launcher\n# -rwxr-xr-x. 1 highland highland  4545360 Feb 11 23:08 eosiowd\n```\n\n\n## Start your first node\nPlease refer to [official documents](https://github.com/EOSIO/eos/wiki/Local-Environment#4-creating-and-launching-a-single-node-testnet).",
      "json_metadata": "{\"tags\":[\"eos\",\"centos\",\"compile\"],\"links\":[\"https://steemit.com/eos/@highland0971/how-to-compile-eos-on-clean-centos-system\",\"https://github.com/EOSIO/eos/wiki/Local-Environment#22-manual-build-script\",\"https://github.com/EOSIO/eos/wiki/Local-Environment#2-building-eos\",\"https://github.com/llvm-mirror/llvm.git\",\"https://github.com/EOSIO/eos/wiki/Local-Environment#4-creating-and-launching-a-single-node-testnet\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}",
      "parent_author": "",
      "parent_permlink": "eos",
      "permlink": "how-to-compile-eos-on-clean-centos-system-updated",
      "title": "How to compile EOS on clean CentOS system [updated]"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-11T15:37:00",
  "trx_id": "720ea4c5d3062697e53a0dccbfb5222f26f8a868",
  "trx_in_block": 53,
  "virtual_op": 0
}
highland0971received 0.021 SBD, 0.009 SP author reward for @highland0971 / how-to-compile-eos-on-clean-centos-system
2018/02/08 02:04:03
19,677,316|virtual
authorhighland0971
permlinkhow-to-compile-eos-on-clean-centos-system
sbd payout0.021 SBD
steem payout0.000 STEEM
vesting payout14.317069 VESTS
Transaction InfoBlock #19677316/Virtual Operation #4
View Raw JSON Data
{
  "block": 19677316,
  "op": [
    "author_reward",
    {
      "author": "highland0971",
      "permlink": "how-to-compile-eos-on-clean-centos-system",
      "sbd_payout": "0.021 SBD",
      "steem_payout": "0.000 STEEM",
      "vesting_payout": "14.317069 VESTS"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-08T02:04:03",
  "trx_id": "0000000000000000000000000000000000000000",
  "trx_in_block": 4294967295,
  "virtual_op": 4
}
podugoshupvoted (100.00%) @highland0971 / deploy-kubernetes-and-calico-network-in-air-gapped-environment-within-20-minutes
2018/02/02 09:30:30
19,513,725|c48cdaf
authorhighland0971
permlinkdeploy-kubernetes-and-calico-network-in-air-gapped-environment-within-20-minutes
voterpodugosh
weight10000 (100.00%)
Transaction InfoBlock #19513725/Trx c48cdaf78adba664256dceec8a0de671758aa412
View Raw JSON Data
{
  "block": 19513725,
  "op": [
    "vote",
    {
      "author": "highland0971",
      "permlink": "deploy-kubernetes-and-calico-network-in-air-gapped-environment-within-20-minutes",
      "voter": "podugosh",
      "weight": 10000
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-02T09:30:30",
  "trx_id": "c48cdaf78adba664256dceec8a0de671758aa412",
  "trx_in_block": 71,
  "virtual_op": 0
}
reginaignateva5upvoted (100.00%) @highland0971 / asus-merlin-raspberrypi
2018/02/02 01:10:12
19,503,732|12d6ee8
authorhighland0971
permlinkasus-merlin-raspberrypi
voterreginaignateva5
weight10000 (100.00%)
Transaction InfoBlock #19503732/Trx 12d6ee89dd39e11c4eda757f2f694d8909a69f11
View Raw JSON Data
{
  "block": 19503732,
  "op": [
    "vote",
    {
      "author": "highland0971",
      "permlink": "asus-merlin-raspberrypi",
      "voter": "reginaignateva5",
      "weight": 10000
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-02T01:10:12",
  "trx_id": "12d6ee89dd39e11c4eda757f2f694d8909a69f11",
  "trx_in_block": 27,
  "virtual_op": 0
}
highland0971updated their account properties
2018/02/02 00:02:36
19,502,381|ea4ebd8
accounthighland0971
json metadata{"profile":{"name":"Highland","website":"https://highland0971.github.io/","profile_image":"https://upload.jianshu.io/users/upload.jianshu.io/users/upload_avatars/4032965/0d1af180631a?imageMogr2/auto-orient/strip|imageView2/1/w/300/h/300"}}
memo keySTM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp
Transaction InfoBlock #19502381/Trx ea4ebd881248c965a515ac68042c3b0c45b8a5ac
View Raw JSON Data
{
  "block": 19502381,
  "op": [
    "account_update",
    {
      "account": "highland0971",
      "json_metadata": "{\"profile\":{\"name\":\"Highland\",\"website\":\"https://highland0971.github.io/\",\"profile_image\":\"https://upload.jianshu.io/users/upload.jianshu.io/users/upload_avatars/4032965/0d1af180631a?imageMogr2/auto-orient/strip|imageView2/1/w/300/h/300\"}}",
      "memo_key": "STM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-02T00:02:36",
  "trx_id": "ea4ebd881248c965a515ac68042c3b0c45b8a5ac",
  "trx_in_block": 44,
  "virtual_op": 0
}
highland0971published a new post: deploy-kubernetes-and-calico-network-in-air-gapped-environment-within-20-minutes
2018/02/01 23:58:27
19,502,298|f0825af
authorhighland0971
bodyThis is a verified detail installation documents for `K8S 1.9.0` + `Calico 2.6.5` + `Docker 1.12.6` + `etcd 3.2.12` + `kube-dns 1.14.7`. In this article we focus on how to setup a complete functional [Kubernetes](https://kubernetes.io/) cluster with [Calico](https://docs.projectcalico.org/v2.6/introduction/) Network solution and [Docker Private Registry](https://docs.docker.com/registry/deploying/) in an air-gapped environment, which is the common scene for lots of companies. ## 1. Pre-defined environment vars - The following pre-defined env-vars should set on the `Internet available host` (for packages and binaries pre-download) and `INSTALLATION_BASE` host (Which can access air-gapped deployment cluster, for installation launch.) ```bash export DOCKER_CLIENT_USER=highland #For users which need docker CLI access export K8S_NODE01=jin #Kubernetes target deploy node export K8S_NODE02=mu export K8S_NODE03=shui export K8S_NODE04=huo export K8S_NODE01_IP=192.168.122.180 #Kubernetes target deploy node ip export K8S_NODE02_IP=192.168.122.117 export K8S_NODE03_IP=192.168.122.52 export K8S_NODE04_IP=192.168.122.102 export K8S_NODES="$K8S_NODE01 $K8S_NODE02 $K8S_NODE03 $K8S_NODE04" export ETCD_CLIENT_PORT=2379 #ETCD daemon client listen port export ETCD_PEER_PORT=2380 #ETCD daemon peer listen port export ETCD1_IP=$K8S_NODE02_IP #ETCD daemon node ip export ETCD2_IP=$K8S_NODE03_IP export ETCD3_IP=$K8S_NODE04_IP export ETCD_IP_ARRAY=($ETCD1_IP $ETCD2_IP $ETCD3_IP) export ETCD_NODES="$K8S_NODE02 $K8S_NODE03 $K8S_NODE04" export ETCD_NODE_ARRAY=($K8S_NODE02 $K8S_NODE03 $K8S_NODE04) export ETCD_ENDPOINTS=https://$ETCD1_IP:$ETCD_CLIENT_PORT,https://$ETCD2_IP:$ETCD_CLIENT_PORT,https://$ETCD3_IP:$ETCD_CLIENT_PORT export PRIVATE_REGISTRY_SERV=$K8S_NODE02 #Private docker registry deployment node export PRIVATE_REGISTRY_SERV_IP=$K8S_NODE02_IP export PRIVATE_REGISTRY=${PRIVATE_REGISTRY_SERV}:443 #Private docker registry service address export KUBE_MASTER_SERV_01=$K8S_NODE01 #The Non-HA K8S cluster master node export KUBE_MASTER_SERV_IP_01=$K8S_NODE01_IP export KUBE_APISERVER_IP_01=$KUBE_MASTER_SERV_IP_01 #The Non-HA K8S cluster apiserver node export KUBE_APISERVER_PORT=6443 export KUBE_APISERVER=https://$KUBE_APISERVER_IP_01:${KUBE_APISERVER_PORT} export KUBE_CLUSTER_SERVICE_IP=172.16.0.1 #The kubernetes cluster ip address export KUBE_CLUSTER_SERVICE_CIDR=172.16.0.1/16 #The kubernetes cluster CIDR address export KUBE_CLUSTER_POD_CIDR=172.17.0.1/16 #The kubernetes cluster pods CIDR address export CLUSTER_DNS_IP=172.16.0.10 #The kubernetes kube-dns service address export K8S_BASE_VER=v1.9.0 export K8S_DNS_VER=1.14.7 export K8S_PAUSE_VER=3.1 export CALICO_CONTROLLER_VER=v1.0.2 export CALICO_NODE_VER=v2.6.5 export CALICO_CNI_VER=v1.11.2 export K8S_PROXY_VER=${K8S_BASE_VER} export K8S_PAUSE_IMG_NAME=pause-amd64 export K8S_PROXY_IMG_NAME=kube-proxy export K8S_KUBE_DNS_IMG_NAME=k8s-dns-kube-dns-amd64 export K8S_DNSMASQ_IMG_NAME=k8s-dns-dnsmasq-nanny-amd64 export K8S_SIDECAR_IMG_NAME=k8s-dns-sidecar-amd64 export CALICO_CONTROLLER_IMG_NAME=kube-controllers export CALICO_NODE_IMAGE_NAME=node export CALICO_CNI_IMAGE_NAME=cni export GCR_IMAGES="kube-aggregator kube-controller-manager ${K8S_PROXY_IMG_NAME} kube-scheduler kube-apiserver" ``` - Run this on installation base host(**INSTALLATION_BASE**) ```bash #Change the hostname on each node if needed for node in $K8S_NODES ; do ssh $node hostnamectl set-hostname $node done #Setup ssh passwordless login and hostname resolve for node in $K8S_NODES ; do ssh-copy-id -f $node ssh $node "echo $K8S_NODE01_IP $K8S_NODE01 >> /etc/hosts" ssh $node "echo $K8S_NODE02_IP $K8S_NODE02 >> /etc/hosts" ssh $node "echo $K8S_NODE03_IP $K8S_NODE03 >> /etc/hosts" ssh $node "echo $K8S_NODE04_IP $K8S_NODE04 >> /etc/hosts" done ``` ## 2. Download required images and binaries for **K8S**,**Etcd**,**Docker**,**Calico**,**cfssl** - Run the following scripts on Internet available hosts with `Docker` pre-installed ```bash cd mkdir -p all-in-one/bin mkdir -p all-in-one/images mkdir -p all-in-one/rpms mkdir -p all-in-one/mixture mkdir -p all-in-one/configs # For Docker,recommend run the following command on a clean linux host without and packages post installed yum update && yum install -y --downloadonly --downloaddir=all-in-one/rpms docker yum reinstall -y --downloadonly --downloaddir=all-in-one/rpms docker #For kubernetes client & server binary curl -L https://dl.k8s.io/${K8S_BASE_VER}/kubernetes-client-linux-amd64.tar.gz -o all-in-one/mixture/kubernetes-client-linux-amd64.tar.gz curl -L https://dl.k8s.io/${K8S_BASE_VER}/kubernetes-server-linux-amd64.tar.gz -o all-in-one/mixture/kubernetes-server-linux-amd64.tar.gz # For kube-dns for img in $K8S_KUBE_DNS_IMG_NAME $K8S_DNSMASQ_IMG_NAME $K8S_SIDECAR_IMG_NAME ; do echo tar highland0971/${img}:${K8S_DNS_VER} docker pull highland0971/${img}:${K8S_DNS_VER} && \ docker tag highland0971/${img}:${K8S_DNS_VER} ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER} && \ docker save ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER} -o all-in-one/images/${img}:${K8S_DNS_VER}.tar done docker pull highland0971/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} && \ docker tag highland0971/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} ${PRIVATE_REGISTRY}/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} && \ docker save ${PRIVATE_REGISTRY}/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} -o all-in-one/images/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER}.tar # For pause docker pull highland0971/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} && \ docker tag highland0971/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} ${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} && \ docker save ${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} -o all-in-one/images/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER}.tar # For registry docker pull registry && \ docker save docker.io/registry:latest -o all-in-one/images/docker-io.registry.tar # For cfssl tools export CFSSL_URL="https://pkg.cfssl.org/R1.2" curl -L "${CFSSL_URL}/cfssl_linux-amd64" -o all-in-one/bin/cfssl curl -L "${CFSSL_URL}/cfssljson_linux-amd64" -o all-in-one/bin/cfssljson chmod +x all-in-one/bin/cfssl* # For etcd export ETCD_URL="https://github.com/coreos/etcd/releases/download" curl -L "${ETCD_URL}/v3.2.12/etcd-v3.2.12-linux-amd64.tar.gz" -o all-in-one/bin/etcd-v3.2.12-linux-amd64.tar.gz # For Calico export CALICO_URL="https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation" curl -L "${CALICO_URL}/rbac.yaml" -o all-in-one/configs/calico-rbac.yaml curl -L "${CALICO_URL}/hosted/calico.yaml" -o all-in-one/configs/calico-deploy.yaml.conf docker pull quay.io/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} && \ docker tag quay.io/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} ${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} && \ docker save ${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} -o all-in-one/images/calico_${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}.tar docker pull quay.io/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} && \ docker tag quay.io/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} ${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} && \ docker save ${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} -o all-in-one/images/calico_${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}.tar docker pull quay.io/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} && \ docker tag quay.io/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} ${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} && \ docker save ${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} -o all-in-one/images/calico_${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}.tar # Archive and check tar -czf all-in-one.tar.gz all-in-one tar -ztf all-in-one.tar.gz # all-in-one/ # all-in-one/bin/ # all-in-one/bin/cfssl # all-in-one/bin/cfssljson # all-in-one/bin/etcd-v3.2.12-linux-amd64.tar.gz # all-in-one/images/ # all-in-one/images/pause-amd64:3.1.tar # all-in-one/images/docker-io.registry.tar # all-in-one/images/k8s-dns-kube-dns-amd64:1.14.7.tar # all-in-one/images/calico_kube-controllers:v1.0.2.tar # all-in-one/images/calico_node:v2.6.5.tar # all-in-one/images/calico_cni:v1.11.2.tar # all-in-one/images/k8s-dns-dnsmasq-nanny-amd64:1.14.7.tar # all-in-one/images/k8s-dns-sidecar-amd64:1.14.7.tar # all-in-one/rpms/ # all-in-one/rpms/docker-1.12.6-68.gitec8512b.el7.centos.x86_64.rpm # all-in-one/rpms/audit-libs-python-2.7.6-3.el7.x86_64.rpm # all-in-one/rpms/checkpolicy-2.5-4.el7.x86_64.rpm # all-in-one/rpms/container-selinux-2.33-1.git86f33cd.el7.noarch.rpm # all-in-one/rpms/container-storage-setup-0.8.0-3.git1d27ecf.el7.noarch.rpm # all-in-one/rpms/docker-client-1.12.6-68.gitec8512b.el7.centos.x86_64.rpm # all-in-one/rpms/docker-common-1.12.6-68.gitec8512b.el7.centos.x86_64.rpm # all-in-one/rpms/libcgroup-0.41-13.el7.x86_64.rpm # all-in-one/rpms/libsemanage-python-2.5-8.el7.x86_64.rpm # all-in-one/rpms/oci-register-machine-0-3.13.gitcd1e331.el7.x86_64.rpm # all-in-one/rpms/oci-systemd-hook-0.1.14-1.git1ba44c6.el7.x86_64.rpm # all-in-one/rpms/oci-umount-2.3.0-1.git51e7c50.el7.x86_64.rpm # all-in-one/rpms/policycoreutils-python-2.5-17.1.el7.x86_64.rpm # all-in-one/rpms/python-IPy-0.75-6.el7.noarch.rpm # all-in-one/rpms/setools-libs-3.3.8-1.1.el7.x86_64.rpm # all-in-one/rpms/skopeo-containers-0.1.26-2.dev.git2e8377a.el7.centos.x86_64.rpm # all-in-one/rpms/yajl-2.0.4-4.el7.x86_64.rpm # all-in-one/mixture/ # all-in-one/mixture/kubernetes-server-linux-amd64.tar.gz # all-in-one/mixture/kubernetes-client-linux-amd64.tar.gz # all-in-one/configs/ # all-in-one/configs/calico-rbac.yaml # all-in-one/configs/calico-deploy.yaml # all-in-one/configs/calico-deploy.yaml.conf ``` - Distribute to target installation base host You can change the distribution method depends on your environment ```bash # define ${INSTALLATION_BASE} as you wanted scp all-in-one.tar.gz ${INSTALLATION_BASE}:~/ ssh ${INSTALLATION_BASE} tar -zxf ~/all-in-one.tar.gz ``` # The following scripts executed on **INSTALLATION_BASE** ## 3. Prepare kubelet runtime environment ```bash for node in ${K8S_NODES} ;do echo Disable swap file on ${node} ssh ${node} swapoff -a ssh ${node} "grep 'swap' /etc/fstab && cp /etc/fstab /etc/fstab.with_swap && echo orginal /etc/fstab file has been backup to etc/fstab.with_swap " ssh ${node} sed -i '/swap/d' /etc/fstab echo Disable SeLinux on ${node} ssh $node setenforce 0 ssh $node "sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config" echo Disable firewalld on ${node} ssh $node systemctl stop firewalld ssh $node systemctl disable firewalld done ``` ## 4. Install `Docker` Install `Docker` on target hosts ```bash # Scp docker rpm packages to each nodes, passwdless environment must be set previously for root user # Execute current command as root for node in ${K8S_NODES} ; do ssh $node mkdir ~/rpms -p scp all-in-one/rpms/* ${node}:~/rpms ssh $node rpm -ivh ~/rpms/*.rpm ssh $node groupadd docker ssh $node usermod -aG docker $DOCKER_CLIENT_USER ssh $node "sed -i '/^ExecReload/i\ExecStartPost=\/sbin\/iptables -I FORWARD -s 0.0.0.0\/0 -j ACCEPT' /lib/systemd/system/docker.service" ssh $node systemctl disable docker ssh $node systemctl enable docker ssh $node systemctl stop docker ssh $node systemctl start docker done ``` ## 5. Setup private registry service ```bash cd cp all-in-one/bin/cfssl* /usr/local/bin/ scp all-in-one/images/docker-io.registry.tar ${PRIVATE_REGISTRY_SERV}:~/ ssh ${PRIVATE_REGISTRY_SERV} docker load -i ~/docker-io.registry.tar ssh ${PRIVATE_REGISTRY_SERV} mkdir /etc/docker/ssl -p mkdir certs/docker -p #Generate docker CA cat <<EOF > certs/ca-config.json { "signing":{ "default":{ "expiry":"87600h" }, "profiles":{ "default":{ "usages":[ "signing", "key encipherment", "server auth", "client auth" ], "expiry":"87600h" } } } } EOF cat <<EOF > certs/docker/docker-ca-csr.json { "CN":"docker-ca", "key":{ "algo":"rsa", "size":2048 }, "names":[ { "C":"cn", "ST":"DreamState", "L":"DreamCity", "O":"DreamOrganization", "OU":"DockerPrivateRegistry" } ] } EOF cfssl gencert -initca certs/docker/docker-ca-csr.json | cfssljson -bare certs/docker/docker-ca #Generate docker TLS certificates cat <<EOF > certs/docker/private-registry-csr.json { "CN":"private-registry", "hosts":[ "127.0.0.1", "${PRIVATE_REGISTRY_SERV_IP}", "${PRIVATE_REGISTRY_SERV}" ], "key":{ "algo":"rsa", "size":2048 }, "names":[ { "C":"cn", "ST":"DreamState", "L":"DreamCity", "O":"DreamOrganization", "OU":"DockerPrivateRegistry" } ] } EOF cfssl gencert -ca=certs/docker/docker-ca.pem \ -ca-key=certs/docker/docker-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/docker/private-registry-csr.json | cfssljson -bare certs/docker/private-registry #Make private registry trusted in the nodes by copy Docker CA file to /etc/docker/certs.d/${PRIVATE_REGISTRY} for node in ${K8S_NODES} ; do ssh ${node} mkdir -p /etc/docker/certs.d/${PRIVATE_REGISTRY} scp certs/docker/docker-ca.pem ${node}:/etc/docker/certs.d/${PRIVATE_REGISTRY}/ca.crt done #Start the private registry service ssh ${PRIVATE_REGISTRY_SERV} mkdir -p /etc/docker/ssl scp certs/docker/private-registry*.pem ${PRIVATE_REGISTRY_SERV}:/etc/docker/ssl ssh ${PRIVATE_REGISTRY_SERV} mkdir -p /mnt/registry ssh ${PRIVATE_REGISTRY_SERV} "docker run -d --restart=always -v /mnt/registry:/var/lib/registry -v /etc/docker/ssl:/certs -e REGISTRY_HTTP_ADDR=0.0.0.0:443 -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/private-registry.pem -e REGISTRY_HTTP_TLS_KEY=/certs/private-registry-key.pem -p 443:443 --name private-registry registry" ssh ${PRIVATE_REGISTRY_SERV} docker logs private-registry ``` ## 6. Distribute K8s binaries and images to nodes ```bash cd tar -zxf all-in-one/mixture/kubernetes-client-linux-amd64.tar.gz tar -zxf all-in-one/mixture/kubernetes-server-linux-amd64.tar.gz for node in ${K8S_NODES} ; do echo Process node ${node}..... scp kubernetes/server/bin/kubectl ${node}:/usr/local/bin scp kubernetes/server/bin/kubelet ${node}:/usr/local/bin ssh ${node} chmod +x /usr/local/bin/kube* done ssh ${PRIVATE_REGISTRY_SERV} mkdir ~/images scp kubernetes/server/bin/*.tar ${PRIVATE_REGISTRY_SERV}:~/images scp all-in-one/images/*.tar ${PRIVATE_REGISTRY_SERV}:~/images ssh ${PRIVATE_REGISTRY_SERV} docker load -i images/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER}.tar ssh ${PRIVATE_REGISTRY_SERV} docker push ${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} ssh ${PRIVATE_REGISTRY_SERV} docker rmi ${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} for img in $K8S_KUBE_DNS_IMG_NAME $K8S_DNSMASQ_IMG_NAME $K8S_SIDECAR_IMG_NAME ; do echo push ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER} ssh ${PRIVATE_REGISTRY_SERV} docker load -i images/${img}:${K8S_DNS_VER}.tar ssh ${PRIVATE_REGISTRY_SERV} docker push ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER} ssh ${PRIVATE_REGISTRY_SERV} docker rmi ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER} done ssh ${PRIVATE_REGISTRY_SERV} docker load -i images/calico_${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}.tar ssh ${PRIVATE_REGISTRY_SERV} docker push ${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} ssh ${PRIVATE_REGISTRY_SERV} docker rmi ${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} ssh ${PRIVATE_REGISTRY_SERV} docker load -i images/calico_${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}.tar ssh ${PRIVATE_REGISTRY_SERV} docker push ${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} ssh ${PRIVATE_REGISTRY_SERV} docker rmi ${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} ssh ${PRIVATE_REGISTRY_SERV} docker load -i images/calico_${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}.tar ssh ${PRIVATE_REGISTRY_SERV} docker push ${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} ssh ${PRIVATE_REGISTRY_SERV} docker rmi ${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} export gcr=gcr.io/google_containers for img in ${GCR_IMAGES} ; do echo push ${PRIVATE_REGISTRY}/${img}:${K8S_BASE_VER} ssh ${PRIVATE_REGISTRY_SERV} docker load -i ~/images/${img}.tar ssh ${PRIVATE_REGISTRY_SERV} docker tag ${gcr}/${img}:${K8S_BASE_VER} ${PRIVATE_REGISTRY}/${img}:${K8S_BASE_VER} ssh ${PRIVATE_REGISTRY_SERV} docker push ${PRIVATE_REGISTRY}/${img}:${K8S_BASE_VER} ssh ${PRIVATE_REGISTRY_SERV} docker rmi ${gcr}/${img}:${K8S_BASE_VER} ssh ${PRIVATE_REGISTRY_SERV} docker rmi ${PRIVATE_REGISTRY}/${img}:${K8S_BASE_VER} done ssh ${PRIVATE_REGISTRY_SERV} rm -rf images ``` ## 7. Setup `etcd` daemon - prepare certificates ```bash cd mkdir -p certs/etcd cat <<EOF > certs/etcd/etcd-ca-csr.json { "CN":"etcd-ca", "key":{ "algo":"rsa", "size":2048 }, "names":[ { "C":"cn", "ST":"DreamState", "L":"DreamCity", "O":"DreamOrganization", "OU":"etcd" } ] } EOF cfssl gencert -initca certs/etcd/etcd-ca-csr.json | cfssljson -bare certs/etcd/etcd-ca cat <<EOF > certs/etcd/etcd-csr.json { "CN":"etcd", "hosts":[ "127.0.0.1", "${ETCD1_IP}", "${ETCD2_IP}", "${ETCD3_IP}" ], "key":{ "algo":"rsa", "size":2048 }, "names":[ { "C":"cn", "ST":"DreamState", "L":"DreamCity", "O":"DreamOrganization", "OU":"etcd" } ] } EOF cfssl gencert -ca=certs/etcd/etcd-ca.pem \ -ca-key=certs/etcd/etcd-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/etcd/etcd-csr.json | cfssljson -bare certs/etcd/etcd for node in ${ETCD_NODES} ;do echo Copy certs to $node ... ssh ${node} mkdir -p /etc/etcd/ssl scp certs/etcd/* ${node}:/etc/etcd/ssl done ``` - prepare binaries and configs ```bash cd tar -zxf all-in-one/bin/etcd-v3.2.12-linux-amd64.tar.gz let node_idx=0 for node in ${ETCD_NODES} ;do echo process node ${node} ssh ${node} "systemctl stop etcd.service && systemctl disable etcd.service" scp etcd-v3.2.12-linux-amd64/etcd* ${node}:/usr/local/bin ssh ${node} groupadd etcd ssh ${node} "useradd -c 'Etcd user' -g etcd -s /sbin/nologin -r etcd" ssh ${node} mkdir -p /var/lib/etcd ssh ${node} chown etcd:etcd -R /var/lib/etcd /etc/etcd cat << EOF > etcd.conf # [member] ETCD_NAME=${node} ETCD_DATA_DIR=/var/lib/etcd ETCD_LISTEN_PEER_URLS=https://0.0.0.0:${ETCD_PEER_PORT} ETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:${ETCD_CLIENT_PORT} ETCD_PROXY=off # [cluster] ETCD_ADVERTISE_CLIENT_URLS=https://${ETCD_IP_ARRAY[${node_idx}]}:${ETCD_CLIENT_PORT} ETCD_INITIAL_ADVERTISE_PEER_URLS=https://${ETCD_IP_ARRAY[${node_idx}]}:${ETCD_PEER_PORT} ETCD_INITIAL_CLUSTER=${ETCD_NODE_ARRAY[0]}=https://${ETCD_IP_ARRAY[0]}:${ETCD_PEER_PORT},${ETCD_NODE_ARRAY[1]}=https://${ETCD_IP_ARRAY[1]}:${ETCD_PEER_PORT},${ETCD_NODE_ARRAY[2]}=https://${ETCD_IP_ARRAY[2]}:${ETCD_PEER_PORT} ETCD_INITIAL_CLUSTER_STATE=new ETCD_INITIAL_CLUSTER_TOKEN=etcd-k8s-cluster # [security] ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem" ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" ETCD_CLIENT_CERT_AUTH="true" ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem" ETCD_AUTO_TLS="true" ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem" ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem" ETCD_PEER_CLIENT_CERT_AUTH="true" ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem" ETCD_PEER_AUTO_TLS="true" EOF cat <<EOF > etcd.service [Unit] Description=Etcd Service After=network.target [Service] EnvironmentFile=-/etc/etcd/etcd.conf Type=simple User=etcd PermissionsStartOnly=true ExecStart=/usr/local/bin/etcd Restart=on-failure RestartSec=10 LimitNOFILE=65536 [Install] WantedBy=multi-user.target EOF ssh ${node} mkdir -p /etc/etcd ssh ${node} mkdir -p /var/lib/etcd scp etcd.conf ${node}:/etc/etcd scp etcd.service ${node}:/lib/systemd/system ssh ${node} chown etcd:etcd -R /var/lib/etcd /etc/etcd ssh ${node} systemctl enable etcd.service ssh ${node} systemctl start etcd.service ssh ${node} ETCDCTL_API=3 etcdctl --cacert=/etc/etcd/ssl/etcd-ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://${ETCD_IP_ARRAY[${node_idx}]}:${ETCD_CLIENT_PORT}" endpoint health ssh ${node} etcdctl --ca-file=/etc/etcd/ssl/etcd-ca.pem -cert-file=/etc/etcd/ssl/etcd.pem -key-file=/etc/etcd/ssl/etcd-key.pem --endpoints=https://${ETCD_IP_ARRAY[${node_idx}]}:${ETCD_CLIENT_PORT} cluster-health let node_idx=${node_idx}+1 done ``` ## 8. Prepare certificate for `K8s` ```bash cd mkdir -p certs/k8s #For kubernetes root CA file cat <<EOF > certs/k8s/k8s-ca-csr.json { "CN":"kubernetes", "key":{ "algo":"rsa", "size":2048 }, "names":[ { "C":"cn", "ST":"DreamState", "L":"DreamCity", "O":"DreamOrganization", "OU":"DreamSystem" } ] } EOF cfssl gencert -initca certs/k8s/k8s-ca-csr.json | cfssljson -bare certs/k8s/k8s-ca #For API server certificates cat <<EOF > certs/k8s/kube-apiserver-csr.json { "CN":"kube-apiserver", "hosts":[ "127.0.0.1", "${KUBE_MASTER_SERV_IP_01}", "${KUBE_MASTER_SERV_01}", "${KUBE_CLUSTER_SERVICE_IP}", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local" ], "key":{ "algo":"rsa", "size":2048 }, "names":[ { "O":"system:masters" } ] } EOF cfssl gencert -ca=certs/k8s/k8s-ca.pem \ -ca-key=certs/k8s/k8s-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/k8s/kube-apiserver-csr.json | cfssljson -bare certs/k8s/kube-apiserver #For front-proxy-client certificates cat <<EOF > certs/k8s/kube-front-proxy-client-csr.json { "CN":"front-proxy-client", "key":{ "algo":"rsa", "size":2048 } } EOF cfssl gencert -ca=certs/k8s/k8s-ca.pem \ -ca-key=certs/k8s/k8s-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/k8s/kube-front-proxy-client-csr.json | cfssljson -bare certs/k8s/kube-front-proxy-client #For controller-manager certificates cat <<EOF > certs/k8s/kube-controller-manager-csr.json { "CN":"system:kube-controller-manager", "key":{ "algo":"rsa", "size":2048 }, "names":[ { "O":"system:masters" } ] } EOF cfssl gencert -ca=certs/k8s/k8s-ca.pem \ -ca-key=certs/k8s/k8s-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/k8s/kube-controller-manager-csr.json | cfssljson -bare certs/k8s/kube-controller-manager #For scheduler certificates cat <<EOF > certs/k8s/kube-scheduler-csr.json { "CN":"system:kube-scheduler", "key":{ "algo":"rsa", "size":2048 }, "names":[ { "O":"system:masters" } ] } EOF cfssl gencert -ca=certs/k8s/k8s-ca.pem \ -ca-key=certs/k8s/k8s-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/k8s/kube-scheduler-csr.json | cfssljson -bare certs/k8s/kube-scheduler #For kubelet master certificates cat <<EOF > certs/k8s/kubelet-master-csr.json { "CN":"system:node:${KUBE_MASTER_SERV_01}", "key":{ "algo":"rsa", "size":2048 }, "names":[ { "O":"system:nodes" } ] } EOF cfssl gencert -ca=certs/k8s/k8s-ca.pem \ -ca-key=certs/k8s/k8s-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/k8s/kubelet-master-csr.json | cfssljson -bare certs/k8s/kubelet-master #For admin certificate cat <<EOF > certs/k8s/kube-admin-csr.json { "CN":"kube-admin", "key":{ "algo":"rsa", "size":2048 }, "names":[ { "C":"cn", "ST":"DreamState", "L":"DreamCity", "O":"system:masters", "OU":"DreamTeam" } ] } EOF cfssl gencert -ca=certs/k8s/k8s-ca.pem \ -ca-key=certs/k8s/k8s-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/k8s/kube-admin-csr.json | cfssljson -bare certs/k8s/kube-admin #For service account key openssl genrsa -out certs/k8s/sa.key 2048 openssl rsa -in certs/k8s/sa.key -pubout -out certs/k8s/sa.pub #For etcd client cat <<EOF > certs/k8s/kubelet-etcd-client-csr.json { "CN":"kubelet-nodes", "key":{ "algo":"rsa", "size":2048 } } EOF cfssl gencert -ca=certs/etcd/etcd-ca.pem \ -ca-key=certs/etcd/etcd-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/k8s/kubelet-etcd-client-csr.json | cfssljson -bare certs/k8s/kubelet-etcd-client ``` ## 9. Prepare kubeconfig file for K8s cluster - Prepare kubeconfig file for each component ```bash cd # For admin export TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ') cat <<EOF > certs/k8s/token.csv ${TOKEN},kube-admin,10002,"system:masters" EOF kubernetes/server/bin/kubectl config set-cluster kubernetes \ --certificate-authority=certs/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=certs/k8s/admin.conf kubernetes/server/bin/kubectl config set-credentials kubernetes-admin \ --client-certificate=certs/k8s/kube-admin.pem \ --client-key=certs/k8s/kube-admin-key.pem \ --embed-certs=true \ --kubeconfig=certs/k8s/admin.conf kubernetes/server/bin/kubectl config set-credentials kubernetes-admin \ --token=${TOKEN} \ --kubeconfig=certs/k8s/admin.conf kubernetes/server/bin/kubectl config set-context default \ --cluster=kubernetes \ --user=kubernetes-admin \ --kubeconfig=certs/k8s/admin.conf kubernetes/server/bin/kubectl config use-context default \ --kubeconfig=certs/k8s/admin.conf # For bootstrap export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ') cat <<EOF >> certs/k8s/token.csv ${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap" EOF kubernetes/server/bin/kubectl config set-cluster kubernetes \ --certificate-authority=certs/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=certs/k8s/bootstrap.conf kubernetes/server/bin/kubectl config set-credentials kubelet-bootstrap \ --token=${BOOTSTRAP_TOKEN} \ --kubeconfig=certs/k8s/bootstrap.conf kubernetes/server/bin/kubectl config set-context default \ --cluster=kubernetes \ --user=kubelet-bootstrap \ --kubeconfig=certs/k8s/bootstrap.conf kubernetes/server/bin/kubectl config use-context default \ --kubeconfig=certs/k8s/bootstrap.conf # For kubelet master kubernetes/server/bin/kubectl config set-cluster kubernetes \ --certificate-authority=certs/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=certs/k8s/kubelet-master.conf kubernetes/server/bin/kubectl config set-credentials system:node:${KUBE_MASTER_SERV_01} \ --client-certificate=certs/k8s/kubelet-master.pem \ --client-key=certs/k8s/kubelet-master-key.pem \ --embed-certs=true \ --kubeconfig=certs/k8s/kubelet-master.conf kubernetes/server/bin/kubectl config set-context default \ --cluster=kubernetes \ --user=system:node:${KUBE_MASTER_SERV_01} \ --kubeconfig=certs/k8s/kubelet-master.conf kubernetes/server/bin/kubectl config use-context default \ --kubeconfig=certs/k8s/kubelet-master.conf #For controller-manager kubernetes/server/bin/kubectl config set-cluster kubernetes \ --certificate-authority=certs/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=certs/k8s/controller-manager.conf kubernetes/server/bin/kubectl config set-credentials system:kube-controller-manager \ --client-certificate=certs/k8s/kube-controller-manager.pem \ --client-key=certs/k8s/kube-controller-manager-key.pem \ --embed-certs=true \ --kubeconfig=certs/k8s/controller-manager.conf kubernetes/server/bin/kubectl config set-context default \ --cluster=kubernetes \ --user=system:kube-controller-manager \ --kubeconfig=certs/k8s/controller-manager.conf kubernetes/server/bin/kubectl config use-context default \ --kubeconfig=certs/k8s/controller-manager.conf # For scheduler kubernetes/server/bin/kubectl config set-cluster kubernetes \ --certificate-authority=certs/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=certs/k8s/scheduler.conf kubernetes/server/bin/kubectl config set-credentials system:kube-scheduler \ --client-certificate=certs/k8s/kube-scheduler.pem \ --client-key=certs/k8s/kube-scheduler-key.pem \ --embed-certs=true \ --kubeconfig=certs/k8s/scheduler.conf kubernetes/server/bin/kubectl config set-context default \ --cluster=kubernetes \ --user=system:kube-scheduler \ --kubeconfig=certs/k8s/scheduler.conf kubernetes/server/bin/kubectl config use-context default \ --kubeconfig=certs/k8s/scheduler.conf #Check the output in `certs/k8s` ls certs/k8s # admin.conf kube-admin-csr.json kube-controller-manager-csr.json kubelet-etcd-client-csr.json kube-scheduler.csr # bootstrap.conf kube-admin-key.pem kube-controller-manager-key.pem kubelet-etcd-client-key.pem kube-scheduler-csr.json # controller-manager.conf kube-admin.pem kube-controller-manager.pem kubelet-etcd-client.pem kube-scheduler-key.pem # k8s-ca.csr kube-apiserver.csr kube-front-proxy-client.csr kubelet-master.conf kube-scheduler.pem # k8s-ca-csr.json kube-apiserver-csr.json kube-front-proxy-client-csr.json kubelet-master.csr sa.key # k8s-ca-key.pem kube-apiserver-key.pem kube-front-proxy-client-key.pem kubelet-master-csr.json sa.pub # k8s-ca.pem kube-apiserver.pem kube-front-proxy-client.pem kubelet-master-key.pem scheduler.conf # kube-admin.csr kube-controller-manager.csr kubelet-etcd-client.csr kubelet-master.pem token.csv ``` - Distribute certificate and kubeconfig file to master and nodes ```bash #For non-master node, without kubeconfig file,only copy bootstrap.conf for node in ${K8S_NODES} ;do echo Copy node certificate file to $node ssh ${node} mkdir -p /etc/kubernetes/pki for files in \ certs/etcd/etcd-ca.pem \ certs/k8s/kubelet-etcd-client*.pem \ certs/k8s/k8s-ca*.pem ;\ do scp $files ${node}:/etc/kubernetes/pki done scp certs/k8s/bootstrap.conf ${node}:/etc/kubernetes done #For master node,which has --kubeconfig file copied echo Copy node certificate file to ${KUBE_MASTER_SERV_01}[master] for files in \ certs/k8s/*.pem \ certs/k8s/sa.* \ certs/etcd/etcd-ca.pem ; \ do scp $files ${KUBE_MASTER_SERV_01}:/etc/kubernetes/pki done scp certs/k8s/*.conf ${KUBE_MASTER_SERV_01}:/etc/kubernetes scp certs/k8s/token.csv ${KUBE_MASTER_SERV_01}:/etc/kubernetes ``` ## 10. Setup `apiserver` `controller-manager` `scheduler` static pods yaml configs ```bash cd mkdir kubernetes/manifests -p ``` - For `apiserver` pod ```bash cat > kubernetes/manifests/kube-apiserver.yaml << EOF apiVersion: v1 kind: Pod metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" labels: component: kube-apiserver tier: control-plane name: kube-apiserver namespace: kube-system spec: hostNetwork: true containers : - name: kube-apiserver image: ${PRIVATE_REGISTRY}/kube-apiserver:${K8S_BASE_VER} command: - kube-apiserver - --v=0 - --logtostderr=true - --allow-privileged=true - --bind-address=0.0.0.0 - --secure-port=${KUBE_APISERVER_PORT} - --insecure-port=0 - --advertise-address=${KUBE_APISERVER_IP_01} - --service-cluster-ip-range=${KUBE_CLUSTER_SERVICE_CIDR} - --etcd-servers=https://${ETCD1_IP}:${ETCD_CLIENT_PORT},https://${ETCD2_IP}:${ETCD_CLIENT_PORT},https://${ETCD3_IP}:${ETCD_CLIENT_PORT} - --etcd-cafile=/etc/kubernetes/pki/etcd-ca.pem - --etcd-certfile=/etc/kubernetes/pki/kubelet-etcd-client.pem - --etcd-keyfile=/etc/kubernetes/pki/kubelet-etcd-client-key.pem - --client-ca-file=/etc/kubernetes/pki/k8s-ca.pem - --tls-cert-file=/etc/kubernetes/pki/kube-apiserver.pem - --tls-private-key-file=/etc/kubernetes/pki/kube-apiserver-key.pem - --kubelet-client-certificate=/etc/kubernetes/pki/kube-apiserver.pem - --kubelet-client-key=/etc/kubernetes/pki/kube-apiserver-key.pem - --service-account-key-file=/etc/kubernetes/pki/sa.pub - --token-auth-file=/etc/kubernetes/token.csv - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota - --authorization-mode=Node,RBAC - --enable-bootstrap-token-auth=true - --requestheader-client-ca-file=/etc/kubernetes/pki/k8s-ca.pem - --proxy-client-cert-file=/etc/kubernetes/pki/kube-front-proxy-client.pem - --proxy-client-key-file=/etc/kubernetes/pki/kube-front-proxy-client-key.pem #- --requestheader-allowed-names=aggregator - --requestheader-group-headers=X-Remote-Group - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-username-headers=X-Remote-User - --event-ttl=1h livenessProbe: failureThreshold: 8 httpGet: host: 127.0.0.1 path: /healthz port: ${KUBE_APISERVER_PORT} scheme: HTTPS initialDelaySeconds: 15 timeoutSeconds: 15 resources: requests: cpu: 250m volumeMounts: - mountPath: /var/log/kubernetes name: k8s-audit-log - mountPath: /etc/kubernetes/pki name: k8s-certs readOnly: true - mountPath: /etc/kubernetes/token.csv name: token-csv readOnly: true volumes: - hostPath: path: /var/log/kubernetes type: DirectoryOrCreate name: k8s-audit-log - hostPath: path: /etc/kubernetes/pki type: DirectoryOrCreate name: k8s-certs - hostPath: path: /etc/kubernetes/token.csv type: FileOrCreate name: token-csv EOF ``` - For `controller manager` ```bash cat > kubernetes/manifests/kube-controller-manager.yaml << EOF apiVersion: v1 kind: Pod metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" labels: component: kube-controller-manager tier: control-plane name: kube-controller-manager namespace: kube-system spec: hostNetwork: true containers: - name: kube-controller-manager image: ${PRIVATE_REGISTRY}/kube-controller-manager:${K8S_BASE_VER} command: - kube-controller-manager - --v=0 - --logtostderr=true - --address=127.0.0.1 - --root-ca-file=/etc/kubernetes/pki/k8s-ca.pem - --cluster-signing-cert-file=/etc/kubernetes/pki/k8s-ca.pem - --cluster-signing-key-file=/etc/kubernetes/pki/k8s-ca-key.pem - --service-account-private-key-file=/etc/kubernetes/pki/sa.key - --kubeconfig=/etc/kubernetes/controller-manager.conf - --leader-elect=true - --use-service-account-credentials=true - --node-monitor-grace-period=40s - --node-monitor-period=5s - --pod-eviction-timeout=2m0s - --controllers=*,bootstrapsigner,tokencleaner - --allocate-node-cidrs=true - --cluster-cidr=${KUBE_CLUSTER_POD_CIDR} - --node-cidr-mask-size=24 livenessProbe: failureThreshold: 8 httpGet: host: 127.0.0.1 path: /healthz port: 10252 scheme: HTTP initialDelaySeconds: 15 timeoutSeconds: 15 resources: requests: cpu: 200m volumeMounts: - mountPath: /etc/kubernetes/pki name: k8s-certs readOnly: true - mountPath: /etc/ssl/certs name: ca-certs readOnly: true - mountPath: /etc/kubernetes/controller-manager.conf name: kubeconfig readOnly: true - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec name: flexvolume-dir volumes: - hostPath: path: /etc/kubernetes/pki type: DirectoryOrCreate name: k8s-certs - hostPath: path: /etc/ssl/certs type: DirectoryOrCreate name: ca-certs - hostPath: path: /etc/kubernetes/controller-manager.conf type: FileOrCreate name: kubeconfig - hostPath: path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec type: DirectoryOrCreate name: flexvolume-dir EOF ``` - For `scheduler` ```bash cat > kubernetes/manifests/kube-scheduler.yaml << EOF apiVersion: v1 kind: Pod metadata: annotations: scheduler.alpha.kubernetes.io/critical-pod: "" labels: component: kube-scheduler tier: control-plane name: kube-scheduler namespace: kube-system spec: hostNetwork: true containers: - name: kube-scheduler image: ${PRIVATE_REGISTRY}/kube-scheduler:${K8S_BASE_VER} command: - kube-scheduler - --v=0 - --logtostderr=true - --address=127.0.0.1 - --leader-elect=true - --kubeconfig=/etc/kubernetes/scheduler.conf livenessProbe: failureThreshold: 8 httpGet: host: 127.0.0.1 path: /healthz port: 10251 scheme: HTTP initialDelaySeconds: 15 timeoutSeconds: 15 resources: requests: cpu: 100m volumeMounts: - mountPath: /etc/kubernetes/pki name: k8s-certs readOnly: true - mountPath: /etc/kubernetes/scheduler.conf name: kubeconfig readOnly: true volumes: - hostPath: path: /etc/kubernetes/pki type: DirectoryOrCreate name: k8s-certs - hostPath: path: /etc/kubernetes/scheduler.conf type: FileOrCreate name: kubeconfig EOF ``` - Distribute to master node ```bash ssh ${KUBE_MASTER_SERV_01} mkdir -p /etc/kubernetes/manifests scp kubernetes/manifests/*.yaml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/manifests ``` ## 11. Setup and boot the `kubelet` & **cluster** with systemd service ```bash cd mkdir kubernetes/systemd/ -p cat > kubernetes/systemd/kubelet.service << EOF [Unit] Description=kubelet: The Kubernetes Node Agent Documentation=http://kubernetes.io/docs/ [Service] ExecStart=/usr/local/bin/kubelet Restart=on-failure StartLimitInterval=0 RestartSec=10 [Install] WantedBy=multi-user.target EOF cat > kubernetes/systemd/10-kubelet.conf << EOF [Service] Environment="KUBELET_KUBECONFIG_ARGS=--address=0.0.0.0 --port=10250 --kubeconfig=/etc/kubernetes/kubelet-master.conf --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.conf" Environment="KUBE_LOGTOSTDERR=--logtostderr=true --v=0" Environment="KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --anonymous-auth=false" Environment="KUBELET_POD_CONTAINER=--pod-infra-container-image=${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER}" Environment="KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_IP} --cluster-domain=cluster.local" Environment="KUBELET_AUTHZ_ARGS=--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/k8s-ca.pem" Environment="KUBELET_CADVISOR_ARGS=--cadvisor-port=0" Environment="KUBELET_CERTIFICATE_ARGS=--rotate-certificates=true --cert-dir=/var/lib/kubelet/pki" Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false --serialize-image-pulls=false" Environment="KUBE_NODE_LABEL=--node-labels=node-role.kubernetes.io/master=true" #kubelet cgroup-driver MUST be consistence with docker daemon Environment="KUBELET_CGROUP=--cgroup-driver=systemd" ExecStart= ExecStart=/usr/local/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBE_LOGTOSTDERR \$KUBELET_POD_CONTAINER \$KUBELET_SYSTEM_PODS_ARGS \$KUBELET_NETWORK_ARGS \$KUBELET_DNS_ARGS \$KUBELET_AUTHZ_ARGS \$KUBELET_EXTRA_ARGS \$KUBE_NODE_LABEL \$KUBELET_CGROUP EOF for node in ${K8S_NODES} ;do echo Prepare for $node ssh ${node} mkdir -p /etc/systemd/system/kubelet.service.d scp kubernetes/systemd/10-kubelet.conf ${node}:/etc/systemd/system/kubelet.service.d/ scp kubernetes/systemd/kubelet.service ${node}:/lib/systemd/system/kubelet.service ssh ${node} "systemctl stop kubelet && systemctl disable kubelet" ssh ${node} "systemctl enable kubelet && systemctl start kubelet" done ``` ## 12. Setup admin cli environment and check cluster status - Setup the admin cli environment with `amdin.con` file ```bash ssh ${KUBE_MASTER_SERV_01} mkdir ~/.kube ssh ${KUBE_MASTER_SERV_01} cp /etc/kubernetes/admin.conf ~/.kube/config ssh ${KUBE_MASTER_SERV_01} kubectl get cs ssh ${KUBE_MASTER_SERV_01} kubectl get node ssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po ``` - The run out result should be like this: ```bash ssh ${KUBE_MASTER_SERV_01} kubectl get cs # NAME STATUS MESSAGE ERROR # controller-manager Healthy ok # scheduler Healthy ok # etcd-2 Healthy {"health": "true"} # etcd-0 Healthy {"health": "true"} # etcd-1 Healthy {"health": "true"} ssh ${KUBE_MASTER_SERV_01} kubectl get node # NAME STATUS ROLES AGE VERSION # jin NotReady master 1m v1.9.0 ssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po # NAME READY STATUS RESTARTS AGE # kube-apiserver-jin 1/1 Running 0 40s # kube-controller-manager-jin 1/1 Running 0 33s # kube-scheduler-jin 1/1 Running 0 38s ``` ## 13. Authorize other K8s nodes joining the cluster - Add TLS Bootstrapping ClusterRoleBinding to cluster ```bash ssh ${KUBE_MASTER_SERV_01} kubectl create \ clusterrolebinding kubelet-bootstrap \ --clusterrole=system:node-bootstrapper \ --user=kubelet-bootstrap ``` - Check pending node certificate CSR requests ```bash ssh ${KUBE_MASTER_SERV_01} kubectl get csr # NAME AGE REQUESTOR CONDITION # node-csr-gO_HGaj_cJdNzjx3AxjmMtKBNfiBRUdIfBn0HdDyWtM 8s kubelet-bootstrap Pending # node-csr-nrYBLZlNzplcs5pW81BvKTzzenqzXCSQ1IddYFs0BLY 8s kubelet-bootstrap Pending # node-csr-t6TpFDn-uaou9RWRBs0cAXd9V2RIDgretORPWkM0--E 7s kubelet-bootstrap Pending ``` - Authorize node CSR requests ```bash ssh ${KUBE_MASTER_SERV_01} "kubectl get csr | awk '/Pending/ {print \$1}' | xargs kubectl certificate approve" # certificatesigningrequest "node-csr-gO_HGaj_cJdNzjx3AxjmMtKBNfiBRUdIfBn0HdDyWtM" approved # certificatesigningrequest "node-csr-nrYBLZlNzplcs5pW81BvKTzzenqzXCSQ1IddYFs0BLY" approved # certificatesigningrequest "node-csr-t6TpFDn-uaou9RWRBs0cAXd9V2RIDgretORPWkM0--E" approved ssh ${KUBE_MASTER_SERV_01} kubectl get nodes # NAME STATUS ROLES AGE VERSION # huo NotReady master 34s v1.9.0 # jin NotReady master 3m v1.9.0 # mu NotReady master 33s v1.9.0 # shui NotReady master 34s v1.9.0 ``` ## 14. Setup `kube-proxy` addon Which shall be deployed ahead of `Calico` ,otherwise the `Calico` components would stuck in creating container status. ```bash cd mkdir -p kubernetes/addons #Generate kube proxy certificate cat <<EOF > certs/k8s/kube-proxy-csr.json { "CN":"system:kube-proxy", "key":{ "algo":"rsa", "size":2048 }, "names":[ { "O":"system:kube-proxy" } ] } EOF cfssl gencert -ca=certs/k8s/k8s-ca.pem \ -ca-key=certs/k8s/k8s-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/k8s/kube-proxy-csr.json | cfssljson -bare certs/k8s/kube-proxy #Generate kubeconfig for proxy kubernetes/server/bin/kubectl config set-cluster kubernetes \ --certificate-authority=certs/k8s/k8s-ca.pem \ --embed-certs=true \ --server=${KUBE_APISERVER} \ --kubeconfig=certs/k8s/kube-proxy.conf kubernetes/server/bin/kubectl config set-credentials system:kube-proxy \ --client-certificate=certs/k8s/kube-scheduler.pem \ --client-key=certs/k8s/kube-scheduler-key.pem \ --embed-certs=true \ --kubeconfig=certs/k8s/kube-proxy.conf kubernetes/server/bin/kubectl config set-context default \ --cluster=kubernetes \ --user=system:kube-proxy \ --kubeconfig=certs/k8s/kube-proxy.conf kubernetes/server/bin/kubectl config use-context default \ --kubeconfig=certs/k8s/kube-proxy.conf #Generate kube-proxy yaml cat > kubernetes/addons/kube-proxy.yml <<EOF apiVersion: v1 kind: ServiceAccount metadata: name: kube-proxy labels: k8s-app: kube-proxy kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile namespace: kube-system --- apiVersion: extensions/v1beta1 kind: DaemonSet metadata: name: kube-proxy labels: k8s-app: kube-proxy kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile namespace: kube-system spec: selector: matchLabels: k8s-app: kube-proxy templateGeneration: 1 updateStrategy: rollingUpdate: maxUnavailable: 1 type: RollingUpdate template: metadata: labels: k8s-app: kube-proxy annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: serviceAccountName: kube-proxy hostNetwork: true containers: - name: kube-proxy image: ${PRIVATE_REGISTRY}/${K8S_PROXY_IMG_NAME}:${K8S_PROXY_VER} command: - kube-proxy - --v=0 - --logtostderr=true - --kubeconfig=/run/kube-proxy.conf - --cluster-cidr=${KUBE_CLUSTER_POD_CIDR} - --proxy-mode=iptables imagePullPolicy: IfNotPresent securityContext: privileged: true volumeMounts: - mountPath: /run/kube-proxy.conf name: kubeconfig readOnly: true - mountPath: /etc/kubernetes/pki name: k8s-certs readOnly: true dnsPolicy: ClusterFirst restartPolicy: Always terminationGracePeriodSeconds: 30 volumes: - hostPath: path: /etc/kubernetes/kube-proxy.conf type: FileOrCreate name: kubeconfig - hostPath: path: /etc/kubernetes/pki type: DirectoryOrCreate name: k8s-certs EOF #Distribute certs and config file to nodes for node in ${K8S_NODES} ; do scp certs/k8s/kube-proxy*.pem ${node}:/etc/kubernetes/pki scp certs/k8s/kube-proxy.conf ${node}:/etc/kubernetes/ done ssh ${KUBE_MASTER_SERV_01} "mkdir -p /etc/kubernetes/addons" scp kubernetes/addons/kube-proxy.yml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/addons #Apply yaml to cluster ssh ${KUBE_MASTER_SERV_01} kubectl apply -f /etc/kubernetes/addons/kube-proxy.yml #Check status ssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po -l k8s-app=kube-proxy # NAME READY STATUS RESTARTS AGE # kube-proxy-drfbj 1/1 Running 0 45s # kube-proxy-lx6sx 1/1 Running 0 45s # kube-proxy-m44b8 1/1 Running 0 45s # kube-proxy-nb2sw 1/1 Running 0 44s ``` ## 15. `Calico` Standard Hosted Install - Generate certificate for calico-etcd-client ```bash cd mkdir certs/calico -p cat <<EOF > certs/calico/calico-etcd-client-csr.json { "CN":"calico-nodes", "key":{ "algo":"rsa", "size":2048 } } EOF cfssl gencert -ca=certs/etcd/etcd-ca.pem \ -ca-key=certs/etcd/etcd-ca-key.pem \ -config=certs/ca-config.json \ -profile=default \ certs/calico/calico-etcd-client-csr.json | cfssljson -bare certs/calico/calico-etcd-client ``` - Modify preset yaml pattern Setup Calico network in IPIP `OFF` mode,the pattern is fetched from [Standard Hosted Install](https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/hosted) - [calico.yaml](https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/calico.yaml) ```bash export ETCD_CA_BASE64=`cat ${HOME}/certs/etcd/etcd-ca.pem|base64 -w 0` export ETCD_CERT_BASE64=`cat ${HOME}/certs/calico/calico-etcd-client.pem|base64 -w 0` export ETCD_KEY_BASE64=`cat ${HOME}/certs/calico/calico-etcd-client-key.pem|base64 -w 0` sed "s#http://127.0.0.1:2379#$ETCD_ENDPOINTS#" \ all-in-one/configs/calico-deploy.yaml.conf |\ sed 's/etcd_ca: "" #/etcd_ca: /g' |\ sed 's/etcd_cert: "" #/etcd_cert: /g' |\ sed 's/etcd_key: "" #/etcd_key: /g' |\ sed "s/ # etcd-key: null/ etcd-key: $ETCD_KEY_BASE64/" |\ sed "s/ # etcd-cert: null/ etcd-cert: $ETCD_CERT_BASE64/" |\ sed "s/ # etcd-ca: null/ etcd-ca: $ETCD_CA_BASE64/" |\ sed "s#quay.io/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}#${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}#g" |\ sed "s#192.168.0.0/16#${KUBE_CLUSTER_POD_CIDR}#" |\ sed "s#quay.io/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}#${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}#g" |\ sed "s#quay.io/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}#${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}#g" |\ sed "/CALICO_IPV4POOL_IPIP/{n;d}" |\ sed '/CALICO_IPV4POOL_IPIP/a\ value: "off"' > \ all-in-one/configs/calico-deploy.yaml #Distribute config file to master node scp all-in-one/configs/calico-deploy.yaml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/addons scp all-in-one/configs/calico-rbac.yaml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/addons ssh ${KUBE_MASTER_SERV_01} kubectl apply -f /etc/kubernetes/addons/calico-rbac.yaml ssh ${KUBE_MASTER_SERV_01} kubectl apply -f /etc/kubernetes/addons/calico-deploy.yaml #Check the calico pod status ssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po # NAME READY STATUS RESTARTS AGE # calico-kube-controllers-54747bbfbf-rx48m 1/1 Running 0 2m # calico-node-2h6r8 2/2 Running 0 2m # calico-node-gfwzf 2/2 Running 0 2m # calico-node-gvr7v 2/2 Running 0 2m # calico-node-lf4zp 2/2 Running 0 2m # kube-apiserver-jin 1/1 Running 0 16m # kube-controller-manager-jin 1/1 Running 0 16m # kube-proxy-drfbj 1/1 Running 0 12m # kube-proxy-lx6sx 1/1 Running 0 12m # kube-proxy-m44b8 1/1 Running 0 12m # kube-proxy-nb2sw 1/1 Running 0 12m # kube-scheduler-jin 1/1 Running 0 16m ``` ## 16. Setup `kube-dns` addon ```bash cat > kubernetes/addons/kube-dns.yml << EOF apiVersion: v1 kind: ServiceAccount metadata: name: kube-dns labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile namespace: kube-system --- apiVersion: v1 kind: Service metadata: name: kube-dns namespace: kube-system labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: selector: k8s-app: kube-dns clusterIP: ${CLUSTER_DNS_IP} ports: - name: dns port: 53 protocol: UDP - name: dns-tcp port: 53 protocol: TCP --- apiVersion: extensions/v1beta1 kind: Deployment metadata: name: kube-dns namespace: kube-system labels: k8s-app: kube-dns kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: strategy: rollingUpdate: maxSurge: 10% maxUnavailable: 0 selector: matchLabels: k8s-app: kube-dns template: metadata: labels: k8s-app: kube-dns annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: dnsPolicy: Default serviceAccountName: kube-dns tolerations: - key: "CriticalAddonsOnly" operator: "Exists" - key: node-role.kubernetes.io/master effect: NoSchedule volumes: - name: kube-dns-config configMap: name: kube-dns optional: true containers: - name: kubedns image: ${PRIVATE_REGISTRY}/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} resources: limits: memory: 170Mi requests: cpu: 100m memory: 70Mi livenessProbe: httpGet: path: /healthcheck/kubedns port: 10054 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 readinessProbe: httpGet: path: /readiness port: 8081 scheme: HTTP initialDelaySeconds: 3 timeoutSeconds: 5 args: - "--domain=cluster.local" - --dns-port=10053 - --v=2 env: - name: PROMETHEUS_PORT value: "10055" ports: - containerPort: 10053 name: dns-local protocol: UDP - containerPort: 10053 name: dns-tcp-local protocol: TCP - containerPort: 10055 name: metrics protocol: TCP volumeMounts: - name: kube-dns-config mountPath: /kube-dns-config - name: dnsmasq image: ${PRIVATE_REGISTRY}/${K8S_DNSMASQ_IMG_NAME}:${K8S_DNS_VER} livenessProbe: httpGet: path: /healthcheck/dnsmasq port: 10054 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 args: - "-v=2" - "-logtostderr" - "-configDir=/etc/k8s/dns/dnsmasq-nanny" - "-restartDnsmasq=true" - "--" - "-k" - "--cache-size=1000" - "--log-facility=-" - "--server=/cluster.local/127.0.0.1#10053" - "--server=/in-addr.arpa/127.0.0.1#10053" - "--server=/ip6.arpa/127.0.0.1#10053" ports: - containerPort: 53 name: dns protocol: UDP - containerPort: 53 name: dns-tcp protocol: TCP resources: requests: cpu: 150m memory: 20Mi volumeMounts: - name: kube-dns-config mountPath: /etc/k8s/dns/dnsmasq-nanny - name: sidecar image: ${PRIVATE_REGISTRY}/${K8S_SIDECAR_IMG_NAME}:${K8S_DNS_VER} livenessProbe: httpGet: path: /metrics port: 10054 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 args: - "--v=2" - "--logtostderr" - "--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A" - "--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A" ports: - containerPort: 10054 name: metrics protocol: TCP resources: requests: memory: 20Mi cpu: 10m EOF #Distribute certs and config file to nodes ssh ${KUBE_MASTER_SERV_01} mkdir -p /etc/kubernetes/addons scp kubernetes/addons/kube-dns.yml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/addons #Apply yaml to cluster ssh ${KUBE_MASTER_SERV_01} kubectl apply -f /etc/kubernetes/addons/kube-dns.yml #Check DNS status with cluster service IP ssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po -l k8s-app=kube-dns -o wide # NAME READY STATUS RESTARTS AGE IP NODE # kube-dns-c84c85bb4-svv9v 3/3 Running 0 2m 172.17.95.65 mu ``` ## 17. Check the whole cluster status That's all for the cluster deployments, now you can check the overall status in the cluster ```bash ssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get all # NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE # ds/calico-node 4 4 4 4 4 <none> 5m # ds/kube-proxy 4 4 4 4 4 <none> 16m # # NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE # deploy/calico-kube-controllers 1 1 1 1 5m # deploy/calico-policy-controller 0 0 0 0 5m # deploy/kube-dns 1 1 1 1 1m # # NAME DESIRED CURRENT READY AGE # rs/calico-kube-controllers-54747bbfbf 1 1 1 5m # rs/calico-policy-controller-6d964cc58f 0 0 0 5m # rs/kube-dns-c84c85bb4 1 1 1 1m # # NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE # ds/calico-node 4 4 4 4 4 <none> 5m # ds/kube-proxy 4 4 4 4 4 <none> 16m # # NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE # deploy/calico-kube-controllers 1 1 1 1 5m # deploy/calico-policy-controller 0 0 0 0 5m # deploy/kube-dns 1 1 1 1 1m # # NAME DESIRED CURRENT READY AGE # rs/calico-kube-controllers-54747bbfbf 1 1 1 5m # rs/calico-policy-controller-6d964cc58f 0 0 0 5m # rs/kube-dns-c84c85bb4 1 1 1 1m # # NAME READY STATUS RESTARTS AGE # po/calico-kube-controllers-54747bbfbf-rx48m 1/1 Running 0 5m # po/calico-node-2h6r8 2/2 Running 0 5m # po/calico-node-gfwzf 2/2 Running 0 5m # po/calico-node-gvr7v 2/2 Running 0 5m # po/calico-node-lf4zp 2/2 Running 0 5m # po/kube-apiserver-jin 1/1 Running 0 19m # po/kube-controller-manager-jin 1/1 Running 0 19m # po/kube-dns-c84c85bb4-svv9v 3/3 Running 0 1m # po/kube-proxy-drfbj 1/1 Running 0 16m # po/kube-proxy-lx6sx 1/1 Running 0 16m # po/kube-proxy-m44b8 1/1 Running 0 16m # po/kube-proxy-nb2sw 1/1 Running 0 15m # po/kube-scheduler-jin 1/1 Running 0 19m # # NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE # svc/kube-dns ClusterIP 172.16.0.10 <none> 53/UDP,53/TCP 1m ``` ### References - [Standard Hosted Install](https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/hosted) - [Kubernetes 1.8.x 全手动安装教程](https://www.kubernetes.org.cn/3096.html)
json metadata{"tags":["kubernetes","calico","k8s","cn"],"links":["https://kubernetes.io/","https://docs.projectcalico.org/v2.6/introduction/","https://docs.docker.com/registry/deploying/","https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/hosted","https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/calico.yaml","https://www.kubernetes.org.cn/3096.html"],"app":"steemit/0.1","format":"markdown"}
parent author
parent permlinkkubernetes
permlinkdeploy-kubernetes-and-calico-network-in-air-gapped-environment-within-20-minutes
titleDeploy kubernetes & Calico Network in air-gapped environment within 20 minutes
Transaction InfoBlock #19502298/Trx f0825aff1e664fbf877b9ed63547cb54bcbd4d72
View Raw JSON Data
{
  "block": 19502298,
  "op": [
    "comment",
    {
      "author": "highland0971",
      "body": "This is a verified detail installation documents for `K8S 1.9.0` + `Calico 2.6.5` + `Docker 1.12.6` + `etcd 3.2.12` + `kube-dns 1.14.7`.\n\nIn this article we focus on how to setup a complete functional [Kubernetes](https://kubernetes.io/) cluster with [Calico](https://docs.projectcalico.org/v2.6/introduction/) Network solution and [Docker Private Registry](https://docs.docker.com/registry/deploying/) in an air-gapped environment, which is the common scene for lots of companies.\n\n## 1. Pre-defined environment vars\n- The following pre-defined env-vars should set on the `Internet available host` (for packages and binaries pre-download) and `INSTALLATION_BASE` host (Which can access air-gapped deployment cluster, for installation launch.)\n\n```bash\nexport DOCKER_CLIENT_USER=highland #For users which need docker CLI access\nexport K8S_NODE01=jin #Kubernetes target deploy node\nexport K8S_NODE02=mu\nexport K8S_NODE03=shui\nexport K8S_NODE04=huo\nexport K8S_NODE01_IP=192.168.122.180 #Kubernetes target deploy node ip\nexport K8S_NODE02_IP=192.168.122.117\nexport K8S_NODE03_IP=192.168.122.52\nexport K8S_NODE04_IP=192.168.122.102\nexport K8S_NODES=\"$K8S_NODE01 $K8S_NODE02 $K8S_NODE03 $K8S_NODE04\"\nexport ETCD_CLIENT_PORT=2379 #ETCD daemon client listen port\nexport ETCD_PEER_PORT=2380  #ETCD daemon peer listen port\nexport ETCD1_IP=$K8S_NODE02_IP  #ETCD daemon node ip\nexport ETCD2_IP=$K8S_NODE03_IP\nexport ETCD3_IP=$K8S_NODE04_IP\nexport ETCD_IP_ARRAY=($ETCD1_IP $ETCD2_IP $ETCD3_IP)\nexport ETCD_NODES=\"$K8S_NODE02 $K8S_NODE03 $K8S_NODE04\"\nexport ETCD_NODE_ARRAY=($K8S_NODE02 $K8S_NODE03 $K8S_NODE04)\nexport ETCD_ENDPOINTS=https://$ETCD1_IP:$ETCD_CLIENT_PORT,https://$ETCD2_IP:$ETCD_CLIENT_PORT,https://$ETCD3_IP:$ETCD_CLIENT_PORT\nexport PRIVATE_REGISTRY_SERV=$K8S_NODE02 #Private docker registry deployment node\nexport PRIVATE_REGISTRY_SERV_IP=$K8S_NODE02_IP\nexport PRIVATE_REGISTRY=${PRIVATE_REGISTRY_SERV}:443 #Private docker registry service address\nexport KUBE_MASTER_SERV_01=$K8S_NODE01 #The Non-HA K8S cluster master node\nexport KUBE_MASTER_SERV_IP_01=$K8S_NODE01_IP\nexport KUBE_APISERVER_IP_01=$KUBE_MASTER_SERV_IP_01 #The Non-HA K8S cluster apiserver node\nexport KUBE_APISERVER_PORT=6443\nexport KUBE_APISERVER=https://$KUBE_APISERVER_IP_01:${KUBE_APISERVER_PORT}\nexport KUBE_CLUSTER_SERVICE_IP=172.16.0.1  #The kubernetes cluster ip address\nexport KUBE_CLUSTER_SERVICE_CIDR=172.16.0.1/16  #The kubernetes cluster CIDR address\nexport KUBE_CLUSTER_POD_CIDR=172.17.0.1/16  #The kubernetes cluster pods CIDR address\nexport CLUSTER_DNS_IP=172.16.0.10 #The kubernetes kube-dns service address\n\nexport K8S_BASE_VER=v1.9.0\nexport K8S_DNS_VER=1.14.7\nexport K8S_PAUSE_VER=3.1\nexport CALICO_CONTROLLER_VER=v1.0.2\nexport CALICO_NODE_VER=v2.6.5\nexport CALICO_CNI_VER=v1.11.2\n\nexport K8S_PROXY_VER=${K8S_BASE_VER}\nexport K8S_PAUSE_IMG_NAME=pause-amd64\nexport K8S_PROXY_IMG_NAME=kube-proxy\nexport K8S_KUBE_DNS_IMG_NAME=k8s-dns-kube-dns-amd64\nexport K8S_DNSMASQ_IMG_NAME=k8s-dns-dnsmasq-nanny-amd64\nexport K8S_SIDECAR_IMG_NAME=k8s-dns-sidecar-amd64\n\nexport CALICO_CONTROLLER_IMG_NAME=kube-controllers\nexport CALICO_NODE_IMAGE_NAME=node\nexport CALICO_CNI_IMAGE_NAME=cni\n\nexport GCR_IMAGES=\"kube-aggregator kube-controller-manager ${K8S_PROXY_IMG_NAME} kube-scheduler kube-apiserver\"\n```\n\n- Run this on installation base host(**INSTALLATION_BASE**)\n\n```bash\n#Change the hostname on each node if needed\nfor node in $K8S_NODES ; do\n  ssh $node hostnamectl set-hostname $node\ndone\n\n#Setup ssh passwordless login and hostname resolve\nfor node in $K8S_NODES ; do\n  ssh-copy-id -f $node\n  ssh $node \"echo $K8S_NODE01_IP $K8S_NODE01 >> /etc/hosts\"\n  ssh $node \"echo $K8S_NODE02_IP $K8S_NODE02 >> /etc/hosts\"\n  ssh $node \"echo $K8S_NODE03_IP $K8S_NODE03 >> /etc/hosts\"\n  ssh $node \"echo $K8S_NODE04_IP $K8S_NODE04 >> /etc/hosts\"\ndone\n```\n\n## 2. Download required images and binaries for **K8S**,**Etcd**,**Docker**,**Calico**,**cfssl**\n- Run the following scripts on Internet available hosts with `Docker` pre-installed\n\n```bash\ncd\nmkdir -p all-in-one/bin\nmkdir -p all-in-one/images\nmkdir -p all-in-one/rpms\nmkdir -p all-in-one/mixture\nmkdir -p all-in-one/configs\n\n# For Docker,recommend run the following command on a clean linux host without and packages post installed\nyum update && yum install -y --downloadonly  --downloaddir=all-in-one/rpms docker\nyum reinstall -y --downloadonly --downloaddir=all-in-one/rpms docker\n\n#For kubernetes client & server binary\ncurl -L https://dl.k8s.io/${K8S_BASE_VER}/kubernetes-client-linux-amd64.tar.gz -o all-in-one/mixture/kubernetes-client-linux-amd64.tar.gz\ncurl -L https://dl.k8s.io/${K8S_BASE_VER}/kubernetes-server-linux-amd64.tar.gz -o all-in-one/mixture/kubernetes-server-linux-amd64.tar.gz\n\n# For kube-dns\nfor img in $K8S_KUBE_DNS_IMG_NAME $K8S_DNSMASQ_IMG_NAME $K8S_SIDECAR_IMG_NAME ; do\n  echo tar highland0971/${img}:${K8S_DNS_VER}\n  docker pull highland0971/${img}:${K8S_DNS_VER} && \\\n  docker tag highland0971/${img}:${K8S_DNS_VER} ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER} && \\\n  docker save ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER} -o all-in-one/images/${img}:${K8S_DNS_VER}.tar\ndone\n\ndocker pull highland0971/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} && \\\ndocker tag highland0971/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} ${PRIVATE_REGISTRY}/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} && \\\ndocker save ${PRIVATE_REGISTRY}/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER} -o all-in-one/images/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER}.tar\n\n# For pause\ndocker pull highland0971/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} && \\\ndocker tag highland0971/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} ${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} && \\\ndocker save ${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER} -o all-in-one/images/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER}.tar\n\n# For registry\ndocker pull registry && \\\ndocker save docker.io/registry:latest -o all-in-one/images/docker-io.registry.tar\n\n# For cfssl tools\nexport CFSSL_URL=\"https://pkg.cfssl.org/R1.2\"\ncurl -L \"${CFSSL_URL}/cfssl_linux-amd64\" -o all-in-one/bin/cfssl\ncurl -L \"${CFSSL_URL}/cfssljson_linux-amd64\" -o all-in-one/bin/cfssljson\nchmod +x all-in-one/bin/cfssl*\n\n# For etcd\nexport ETCD_URL=\"https://github.com/coreos/etcd/releases/download\"\ncurl -L \"${ETCD_URL}/v3.2.12/etcd-v3.2.12-linux-amd64.tar.gz\" -o all-in-one/bin/etcd-v3.2.12-linux-amd64.tar.gz\n\n# For Calico\nexport CALICO_URL=\"https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation\"\ncurl -L \"${CALICO_URL}/rbac.yaml\" -o all-in-one/configs/calico-rbac.yaml\ncurl -L \"${CALICO_URL}/hosted/calico.yaml\" -o all-in-one/configs/calico-deploy.yaml.conf\n\ndocker pull quay.io/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} && \\\ndocker tag quay.io/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} ${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} && \\\ndocker save ${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER} -o all-in-one/images/calico_${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}.tar\n\ndocker pull quay.io/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} && \\\ndocker tag quay.io/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} ${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} && \\\ndocker save ${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER} -o all-in-one/images/calico_${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}.tar\n\ndocker pull quay.io/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} && \\\ndocker tag quay.io/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} ${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} && \\\ndocker save ${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER} -o all-in-one/images/calico_${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}.tar\n\n# Archive and check\ntar -czf all-in-one.tar.gz all-in-one\ntar -ztf all-in-one.tar.gz\n# all-in-one/\n# all-in-one/bin/\n# all-in-one/bin/cfssl\n# all-in-one/bin/cfssljson\n# all-in-one/bin/etcd-v3.2.12-linux-amd64.tar.gz\n# all-in-one/images/\n# all-in-one/images/pause-amd64:3.1.tar\n# all-in-one/images/docker-io.registry.tar\n# all-in-one/images/k8s-dns-kube-dns-amd64:1.14.7.tar\n# all-in-one/images/calico_kube-controllers:v1.0.2.tar\n# all-in-one/images/calico_node:v2.6.5.tar\n# all-in-one/images/calico_cni:v1.11.2.tar\n# all-in-one/images/k8s-dns-dnsmasq-nanny-amd64:1.14.7.tar\n# all-in-one/images/k8s-dns-sidecar-amd64:1.14.7.tar\n# all-in-one/rpms/\n# all-in-one/rpms/docker-1.12.6-68.gitec8512b.el7.centos.x86_64.rpm\n# all-in-one/rpms/audit-libs-python-2.7.6-3.el7.x86_64.rpm\n# all-in-one/rpms/checkpolicy-2.5-4.el7.x86_64.rpm\n# all-in-one/rpms/container-selinux-2.33-1.git86f33cd.el7.noarch.rpm\n# all-in-one/rpms/container-storage-setup-0.8.0-3.git1d27ecf.el7.noarch.rpm\n# all-in-one/rpms/docker-client-1.12.6-68.gitec8512b.el7.centos.x86_64.rpm\n# all-in-one/rpms/docker-common-1.12.6-68.gitec8512b.el7.centos.x86_64.rpm\n# all-in-one/rpms/libcgroup-0.41-13.el7.x86_64.rpm\n# all-in-one/rpms/libsemanage-python-2.5-8.el7.x86_64.rpm\n# all-in-one/rpms/oci-register-machine-0-3.13.gitcd1e331.el7.x86_64.rpm\n# all-in-one/rpms/oci-systemd-hook-0.1.14-1.git1ba44c6.el7.x86_64.rpm\n# all-in-one/rpms/oci-umount-2.3.0-1.git51e7c50.el7.x86_64.rpm\n# all-in-one/rpms/policycoreutils-python-2.5-17.1.el7.x86_64.rpm\n# all-in-one/rpms/python-IPy-0.75-6.el7.noarch.rpm\n# all-in-one/rpms/setools-libs-3.3.8-1.1.el7.x86_64.rpm\n# all-in-one/rpms/skopeo-containers-0.1.26-2.dev.git2e8377a.el7.centos.x86_64.rpm\n# all-in-one/rpms/yajl-2.0.4-4.el7.x86_64.rpm\n# all-in-one/mixture/\n# all-in-one/mixture/kubernetes-server-linux-amd64.tar.gz\n# all-in-one/mixture/kubernetes-client-linux-amd64.tar.gz\n# all-in-one/configs/\n# all-in-one/configs/calico-rbac.yaml\n# all-in-one/configs/calico-deploy.yaml\n# all-in-one/configs/calico-deploy.yaml.conf\n```\n\n- Distribute to target installation base host\nYou can change the distribution method depends on your environment\n\n```bash\n# define ${INSTALLATION_BASE} as you wanted\nscp all-in-one.tar.gz ${INSTALLATION_BASE}:~/\nssh ${INSTALLATION_BASE} tar -zxf ~/all-in-one.tar.gz\n```\n\n# The following scripts executed on **INSTALLATION_BASE**\n\n## 3. Prepare kubelet runtime environment\n\n```bash\nfor node in ${K8S_NODES} ;do\n  echo Disable swap file on ${node}\n  ssh ${node} swapoff -a\n  ssh ${node} \"grep 'swap' /etc/fstab && cp /etc/fstab /etc/fstab.with_swap && echo orginal /etc/fstab file has been backup to etc/fstab.with_swap \"\n  ssh ${node} sed -i '/swap/d' /etc/fstab\n\n  echo Disable SeLinux on ${node}\n  ssh $node setenforce 0\n  ssh $node \"sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config\"\n\n  echo Disable firewalld on ${node}\n  ssh $node systemctl stop firewalld\n  ssh $node systemctl disable firewalld\ndone\n```\n\n## 4. Install `Docker`\nInstall `Docker` on target hosts\n\n```bash\n# Scp docker rpm packages to each nodes, passwdless environment must be set previously for root user\n# Execute current command as root\n\nfor node in ${K8S_NODES} ; do\n  ssh $node mkdir ~/rpms -p\n  scp all-in-one/rpms/* ${node}:~/rpms\n  ssh $node rpm -ivh ~/rpms/*.rpm\n  ssh $node groupadd docker\n  ssh $node usermod -aG docker $DOCKER_CLIENT_USER\n  ssh $node \"sed -i '/^ExecReload/i\\ExecStartPost=\\/sbin\\/iptables -I FORWARD -s 0.0.0.0\\/0 -j ACCEPT' /lib/systemd/system/docker.service\"\n  ssh $node systemctl disable docker\n  ssh $node systemctl enable docker\n  ssh $node systemctl stop docker\n  ssh $node systemctl start docker\ndone\n```\n## 5. Setup private registry service\n\n```bash\ncd\ncp all-in-one/bin/cfssl* /usr/local/bin/\nscp all-in-one/images/docker-io.registry.tar ${PRIVATE_REGISTRY_SERV}:~/\nssh ${PRIVATE_REGISTRY_SERV} docker load -i ~/docker-io.registry.tar\nssh ${PRIVATE_REGISTRY_SERV} mkdir /etc/docker/ssl -p\n\nmkdir certs/docker -p\n#Generate docker CA\ncat <<EOF > certs/ca-config.json\n{\n  \"signing\":{\n    \"default\":{\n      \"expiry\":\"87600h\"\n    },\n    \"profiles\":{\n      \"default\":{\n        \"usages\":[\n        \"signing\",\n        \"key encipherment\",\n        \"server auth\",\n        \"client auth\"\n        ],\n        \"expiry\":\"87600h\"\n      }\n    }\n  }\n}\nEOF\n\ncat <<EOF > certs/docker/docker-ca-csr.json\n{\n  \"CN\":\"docker-ca\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"C\":\"cn\",\n      \"ST\":\"DreamState\",\n      \"L\":\"DreamCity\",\n      \"O\":\"DreamOrganization\",\n      \"OU\":\"DockerPrivateRegistry\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -initca certs/docker/docker-ca-csr.json | cfssljson -bare certs/docker/docker-ca\n\n#Generate docker TLS certificates\ncat <<EOF > certs/docker/private-registry-csr.json\n{\n  \"CN\":\"private-registry\",\n  \"hosts\":[\n    \"127.0.0.1\",\n    \"${PRIVATE_REGISTRY_SERV_IP}\",\n    \"${PRIVATE_REGISTRY_SERV}\"    \n  ],\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"C\":\"cn\",\n      \"ST\":\"DreamState\",\n      \"L\":\"DreamCity\",\n      \"O\":\"DreamOrganization\",\n      \"OU\":\"DockerPrivateRegistry\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -ca=certs/docker/docker-ca.pem \\\n  -ca-key=certs/docker/docker-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/docker/private-registry-csr.json | cfssljson -bare certs/docker/private-registry\n\n#Make private registry trusted in the nodes by copy Docker CA file to /etc/docker/certs.d/${PRIVATE_REGISTRY}\nfor node in ${K8S_NODES} ; do\n  ssh ${node} mkdir -p /etc/docker/certs.d/${PRIVATE_REGISTRY}\n  scp certs/docker/docker-ca.pem ${node}:/etc/docker/certs.d/${PRIVATE_REGISTRY}/ca.crt\ndone\n\n#Start the private registry service\nssh ${PRIVATE_REGISTRY_SERV} mkdir -p /etc/docker/ssl\nscp certs/docker/private-registry*.pem ${PRIVATE_REGISTRY_SERV}:/etc/docker/ssl\nssh ${PRIVATE_REGISTRY_SERV} mkdir -p /mnt/registry\nssh ${PRIVATE_REGISTRY_SERV} \"docker run -d --restart=always -v /mnt/registry:/var/lib/registry -v /etc/docker/ssl:/certs -e REGISTRY_HTTP_ADDR=0.0.0.0:443 -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/private-registry.pem -e REGISTRY_HTTP_TLS_KEY=/certs/private-registry-key.pem -p 443:443 --name private-registry registry\"\nssh ${PRIVATE_REGISTRY_SERV} docker logs private-registry\n```\n\n## 6. Distribute K8s binaries and images to nodes\n\n```bash\ncd\n\ntar -zxf all-in-one/mixture/kubernetes-client-linux-amd64.tar.gz\ntar -zxf all-in-one/mixture/kubernetes-server-linux-amd64.tar.gz\n\nfor node in ${K8S_NODES} ; do\n  echo Process node ${node}.....\n  scp kubernetes/server/bin/kubectl ${node}:/usr/local/bin\n  scp kubernetes/server/bin/kubelet ${node}:/usr/local/bin\n  ssh ${node} chmod +x /usr/local/bin/kube*\ndone\n\nssh ${PRIVATE_REGISTRY_SERV} mkdir ~/images\nscp kubernetes/server/bin/*.tar ${PRIVATE_REGISTRY_SERV}:~/images\nscp all-in-one/images/*.tar ${PRIVATE_REGISTRY_SERV}:~/images\n\nssh ${PRIVATE_REGISTRY_SERV} docker load -i images/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER}.tar\nssh ${PRIVATE_REGISTRY_SERV} docker push  ${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER}\nssh ${PRIVATE_REGISTRY_SERV} docker rmi  ${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER}\n\nfor img in $K8S_KUBE_DNS_IMG_NAME $K8S_DNSMASQ_IMG_NAME $K8S_SIDECAR_IMG_NAME ; do\n  echo push ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER}\n  ssh ${PRIVATE_REGISTRY_SERV} docker load -i images/${img}:${K8S_DNS_VER}.tar\n  ssh ${PRIVATE_REGISTRY_SERV} docker push  ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER}\n  ssh ${PRIVATE_REGISTRY_SERV} docker rmi  ${PRIVATE_REGISTRY}/${img}:${K8S_DNS_VER}\ndone\n\nssh ${PRIVATE_REGISTRY_SERV} docker load -i images/calico_${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}.tar\nssh ${PRIVATE_REGISTRY_SERV} docker push  ${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}\nssh ${PRIVATE_REGISTRY_SERV} docker rmi  ${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}\n\nssh ${PRIVATE_REGISTRY_SERV} docker load -i images/calico_${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}.tar\nssh ${PRIVATE_REGISTRY_SERV} docker push  ${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}\nssh ${PRIVATE_REGISTRY_SERV} docker rmi  ${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}\n\nssh ${PRIVATE_REGISTRY_SERV} docker load -i images/calico_${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}.tar\nssh ${PRIVATE_REGISTRY_SERV} docker push  ${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}\nssh ${PRIVATE_REGISTRY_SERV} docker rmi  ${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}\n\nexport gcr=gcr.io/google_containers\nfor img in ${GCR_IMAGES} ; do\n  echo push ${PRIVATE_REGISTRY}/${img}:${K8S_BASE_VER}\n  ssh ${PRIVATE_REGISTRY_SERV} docker load -i ~/images/${img}.tar\n  ssh ${PRIVATE_REGISTRY_SERV} docker tag ${gcr}/${img}:${K8S_BASE_VER}  ${PRIVATE_REGISTRY}/${img}:${K8S_BASE_VER}\n  ssh ${PRIVATE_REGISTRY_SERV} docker push  ${PRIVATE_REGISTRY}/${img}:${K8S_BASE_VER}\n  ssh ${PRIVATE_REGISTRY_SERV} docker rmi ${gcr}/${img}:${K8S_BASE_VER}\n  ssh ${PRIVATE_REGISTRY_SERV} docker rmi  ${PRIVATE_REGISTRY}/${img}:${K8S_BASE_VER}\ndone\n\nssh ${PRIVATE_REGISTRY_SERV} rm -rf images\n```\n\n## 7. Setup `etcd` daemon\n- prepare certificates\n\n```bash\ncd\nmkdir -p certs/etcd\ncat <<EOF > certs/etcd/etcd-ca-csr.json\n{\n  \"CN\":\"etcd-ca\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"C\":\"cn\",\n      \"ST\":\"DreamState\",\n      \"L\":\"DreamCity\",\n      \"O\":\"DreamOrganization\",\n      \"OU\":\"etcd\"\n    }\n  ]\n}\nEOF\ncfssl gencert -initca certs/etcd/etcd-ca-csr.json | cfssljson -bare certs/etcd/etcd-ca\n\ncat <<EOF > certs/etcd/etcd-csr.json\n{\n  \"CN\":\"etcd\",\n  \"hosts\":[\n    \"127.0.0.1\",\n    \"${ETCD1_IP}\",\n    \"${ETCD2_IP}\",\n    \"${ETCD3_IP}\"\n  ],\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"C\":\"cn\",\n      \"ST\":\"DreamState\",\n      \"L\":\"DreamCity\",\n      \"O\":\"DreamOrganization\",\n      \"OU\":\"etcd\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -ca=certs/etcd/etcd-ca.pem \\\n  -ca-key=certs/etcd/etcd-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/etcd/etcd-csr.json | cfssljson -bare certs/etcd/etcd\n\nfor node in ${ETCD_NODES} ;do\n  echo Copy certs to $node ...\n  ssh ${node} mkdir -p /etc/etcd/ssl\n  scp certs/etcd/* ${node}:/etc/etcd/ssl\ndone\n```\n\n- prepare binaries and configs\n\n```bash\ncd\ntar -zxf  all-in-one/bin/etcd-v3.2.12-linux-amd64.tar.gz\n\nlet node_idx=0\nfor node in ${ETCD_NODES} ;do\n  echo process node ${node}\n  ssh ${node} \"systemctl stop etcd.service && systemctl disable etcd.service\"\n  scp etcd-v3.2.12-linux-amd64/etcd* ${node}:/usr/local/bin\n  ssh ${node} groupadd etcd\n  ssh ${node} \"useradd -c 'Etcd user' -g etcd -s /sbin/nologin -r etcd\"\n  ssh ${node} mkdir -p /var/lib/etcd\n  ssh ${node} chown etcd:etcd -R /var/lib/etcd /etc/etcd\n  cat << EOF > etcd.conf\n# [member]\nETCD_NAME=${node}\nETCD_DATA_DIR=/var/lib/etcd\nETCD_LISTEN_PEER_URLS=https://0.0.0.0:${ETCD_PEER_PORT}\nETCD_LISTEN_CLIENT_URLS=https://0.0.0.0:${ETCD_CLIENT_PORT}\nETCD_PROXY=off\n\n# [cluster]\nETCD_ADVERTISE_CLIENT_URLS=https://${ETCD_IP_ARRAY[${node_idx}]}:${ETCD_CLIENT_PORT}\nETCD_INITIAL_ADVERTISE_PEER_URLS=https://${ETCD_IP_ARRAY[${node_idx}]}:${ETCD_PEER_PORT}\nETCD_INITIAL_CLUSTER=${ETCD_NODE_ARRAY[0]}=https://${ETCD_IP_ARRAY[0]}:${ETCD_PEER_PORT},${ETCD_NODE_ARRAY[1]}=https://${ETCD_IP_ARRAY[1]}:${ETCD_PEER_PORT},${ETCD_NODE_ARRAY[2]}=https://${ETCD_IP_ARRAY[2]}:${ETCD_PEER_PORT}\nETCD_INITIAL_CLUSTER_STATE=new\nETCD_INITIAL_CLUSTER_TOKEN=etcd-k8s-cluster\n\n# [security]\nETCD_CERT_FILE=\"/etc/etcd/ssl/etcd.pem\"\nETCD_KEY_FILE=\"/etc/etcd/ssl/etcd-key.pem\"\nETCD_CLIENT_CERT_AUTH=\"true\"\nETCD_TRUSTED_CA_FILE=\"/etc/etcd/ssl/etcd-ca.pem\"\nETCD_AUTO_TLS=\"true\"\nETCD_PEER_CERT_FILE=\"/etc/etcd/ssl/etcd.pem\"\nETCD_PEER_KEY_FILE=\"/etc/etcd/ssl/etcd-key.pem\"\nETCD_PEER_CLIENT_CERT_AUTH=\"true\"\nETCD_PEER_TRUSTED_CA_FILE=\"/etc/etcd/ssl/etcd-ca.pem\"\nETCD_PEER_AUTO_TLS=\"true\"\nEOF\n\n  cat <<EOF > etcd.service\n[Unit]\nDescription=Etcd Service\nAfter=network.target\n\n[Service]\nEnvironmentFile=-/etc/etcd/etcd.conf\nType=simple\nUser=etcd\nPermissionsStartOnly=true\nExecStart=/usr/local/bin/etcd\nRestart=on-failure\nRestartSec=10\nLimitNOFILE=65536\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\n  ssh ${node} mkdir -p /etc/etcd\n  ssh ${node} mkdir -p /var/lib/etcd\n  scp etcd.conf ${node}:/etc/etcd\n  scp etcd.service ${node}:/lib/systemd/system\n  ssh ${node} chown etcd:etcd -R /var/lib/etcd /etc/etcd\n\n  ssh ${node} systemctl enable etcd.service\n  ssh ${node} systemctl start etcd.service\n\n  ssh ${node} ETCDCTL_API=3 etcdctl --cacert=/etc/etcd/ssl/etcd-ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem  --endpoints=\"https://${ETCD_IP_ARRAY[${node_idx}]}:${ETCD_CLIENT_PORT}\" endpoint health\n  ssh ${node} etcdctl --ca-file=/etc/etcd/ssl/etcd-ca.pem -cert-file=/etc/etcd/ssl/etcd.pem -key-file=/etc/etcd/ssl/etcd-key.pem  --endpoints=https://${ETCD_IP_ARRAY[${node_idx}]}:${ETCD_CLIENT_PORT} cluster-health\n\n  let node_idx=${node_idx}+1\ndone\n```\n\n## 8. Prepare certificate for `K8s`\n\n```bash\ncd\nmkdir -p certs/k8s\n\n#For kubernetes root CA file\ncat <<EOF > certs/k8s/k8s-ca-csr.json\n{\n  \"CN\":\"kubernetes\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"C\":\"cn\",\n      \"ST\":\"DreamState\",\n      \"L\":\"DreamCity\",\n      \"O\":\"DreamOrganization\",\n      \"OU\":\"DreamSystem\"\n    }\n  ]\n}\nEOF\ncfssl gencert -initca certs/k8s/k8s-ca-csr.json | cfssljson -bare certs/k8s/k8s-ca\n\n#For API server certificates\ncat <<EOF > certs/k8s/kube-apiserver-csr.json\n{\n  \"CN\":\"kube-apiserver\",\n  \"hosts\":[\n    \"127.0.0.1\",\n    \"${KUBE_MASTER_SERV_IP_01}\",\n    \"${KUBE_MASTER_SERV_01}\",\n    \"${KUBE_CLUSTER_SERVICE_IP}\",\n    \"kubernetes\",\n    \"kubernetes.default\",\n    \"kubernetes.default.svc\",\n    \"kubernetes.default.svc.cluster\",\n    \"kubernetes.default.svc.cluster.local\"\n  ],\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"O\":\"system:masters\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -ca=certs/k8s/k8s-ca.pem \\\n  -ca-key=certs/k8s/k8s-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/k8s/kube-apiserver-csr.json | cfssljson -bare certs/k8s/kube-apiserver\n\n#For front-proxy-client certificates\ncat <<EOF > certs/k8s/kube-front-proxy-client-csr.json\n{\n  \"CN\":\"front-proxy-client\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  }\n}\nEOF\n\ncfssl gencert -ca=certs/k8s/k8s-ca.pem \\\n  -ca-key=certs/k8s/k8s-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/k8s/kube-front-proxy-client-csr.json | cfssljson -bare certs/k8s/kube-front-proxy-client\n\n#For controller-manager certificates\ncat <<EOF > certs/k8s/kube-controller-manager-csr.json\n{\n  \"CN\":\"system:kube-controller-manager\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"O\":\"system:masters\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -ca=certs/k8s/k8s-ca.pem \\\n  -ca-key=certs/k8s/k8s-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/k8s/kube-controller-manager-csr.json | cfssljson -bare certs/k8s/kube-controller-manager\n\n#For scheduler certificates\ncat <<EOF > certs/k8s/kube-scheduler-csr.json\n{\n  \"CN\":\"system:kube-scheduler\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"O\":\"system:masters\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -ca=certs/k8s/k8s-ca.pem \\\n  -ca-key=certs/k8s/k8s-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/k8s/kube-scheduler-csr.json | cfssljson -bare certs/k8s/kube-scheduler\n\n#For kubelet master certificates\ncat <<EOF > certs/k8s/kubelet-master-csr.json\n{\n  \"CN\":\"system:node:${KUBE_MASTER_SERV_01}\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"O\":\"system:nodes\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -ca=certs/k8s/k8s-ca.pem \\\n  -ca-key=certs/k8s/k8s-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/k8s/kubelet-master-csr.json | cfssljson -bare certs/k8s/kubelet-master\n\n#For admin certificate\ncat <<EOF > certs/k8s/kube-admin-csr.json\n{\n  \"CN\":\"kube-admin\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"C\":\"cn\",\n      \"ST\":\"DreamState\",\n      \"L\":\"DreamCity\",\n      \"O\":\"system:masters\",\n      \"OU\":\"DreamTeam\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -ca=certs/k8s/k8s-ca.pem \\\n  -ca-key=certs/k8s/k8s-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/k8s/kube-admin-csr.json | cfssljson -bare certs/k8s/kube-admin\n\n#For service account key\nopenssl genrsa -out certs/k8s/sa.key 2048\nopenssl rsa -in certs/k8s/sa.key -pubout -out certs/k8s/sa.pub\n\n#For etcd client\ncat <<EOF > certs/k8s/kubelet-etcd-client-csr.json\n{\n  \"CN\":\"kubelet-nodes\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  }\n}\nEOF\n\ncfssl gencert -ca=certs/etcd/etcd-ca.pem \\\n  -ca-key=certs/etcd/etcd-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/k8s/kubelet-etcd-client-csr.json | cfssljson -bare certs/k8s/kubelet-etcd-client\n\n```\n\n## 9. Prepare kubeconfig file for K8s cluster\n- Prepare kubeconfig file for each component\n\n```bash\ncd\n# For admin\nexport TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')\ncat <<EOF > certs/k8s/token.csv\n${TOKEN},kube-admin,10002,\"system:masters\"\nEOF\nkubernetes/server/bin/kubectl config set-cluster kubernetes \\\n--certificate-authority=certs/k8s/k8s-ca.pem \\\n--embed-certs=true \\\n--server=${KUBE_APISERVER} \\\n--kubeconfig=certs/k8s/admin.conf\n\nkubernetes/server/bin/kubectl config set-credentials kubernetes-admin \\\n--client-certificate=certs/k8s/kube-admin.pem \\\n--client-key=certs/k8s/kube-admin-key.pem \\\n--embed-certs=true \\\n--kubeconfig=certs/k8s/admin.conf\n\nkubernetes/server/bin/kubectl config set-credentials kubernetes-admin \\\n--token=${TOKEN} \\\n--kubeconfig=certs/k8s/admin.conf\n\nkubernetes/server/bin/kubectl config set-context default \\\n--cluster=kubernetes \\\n--user=kubernetes-admin \\\n--kubeconfig=certs/k8s/admin.conf\n\nkubernetes/server/bin/kubectl config use-context default \\\n--kubeconfig=certs/k8s/admin.conf\n\n# For bootstrap\nexport BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')\ncat <<EOF >> certs/k8s/token.csv\n${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,\"system:kubelet-bootstrap\"\nEOF\n\nkubernetes/server/bin/kubectl config set-cluster kubernetes \\\n--certificate-authority=certs/k8s/k8s-ca.pem \\\n--embed-certs=true \\\n--server=${KUBE_APISERVER} \\\n--kubeconfig=certs/k8s/bootstrap.conf\n\nkubernetes/server/bin/kubectl config set-credentials kubelet-bootstrap \\\n--token=${BOOTSTRAP_TOKEN} \\\n--kubeconfig=certs/k8s/bootstrap.conf\n\nkubernetes/server/bin/kubectl  config set-context default \\\n--cluster=kubernetes \\\n--user=kubelet-bootstrap \\\n--kubeconfig=certs/k8s/bootstrap.conf\n\nkubernetes/server/bin/kubectl config use-context default \\\n--kubeconfig=certs/k8s/bootstrap.conf\n\n# For kubelet master\nkubernetes/server/bin/kubectl config set-cluster kubernetes \\\n--certificate-authority=certs/k8s/k8s-ca.pem \\\n--embed-certs=true \\\n--server=${KUBE_APISERVER} \\\n--kubeconfig=certs/k8s/kubelet-master.conf\n\nkubernetes/server/bin/kubectl config set-credentials system:node:${KUBE_MASTER_SERV_01} \\\n--client-certificate=certs/k8s/kubelet-master.pem \\\n--client-key=certs/k8s/kubelet-master-key.pem \\\n--embed-certs=true \\\n--kubeconfig=certs/k8s/kubelet-master.conf\n\nkubernetes/server/bin/kubectl config set-context default \\\n--cluster=kubernetes \\\n--user=system:node:${KUBE_MASTER_SERV_01} \\\n--kubeconfig=certs/k8s/kubelet-master.conf\n\nkubernetes/server/bin/kubectl config use-context default \\\n--kubeconfig=certs/k8s/kubelet-master.conf\n\n#For controller-manager\nkubernetes/server/bin/kubectl config set-cluster kubernetes \\\n--certificate-authority=certs/k8s/k8s-ca.pem \\\n--embed-certs=true \\\n--server=${KUBE_APISERVER} \\\n--kubeconfig=certs/k8s/controller-manager.conf\n\nkubernetes/server/bin/kubectl config set-credentials system:kube-controller-manager \\\n--client-certificate=certs/k8s/kube-controller-manager.pem \\\n--client-key=certs/k8s/kube-controller-manager-key.pem \\\n--embed-certs=true \\\n--kubeconfig=certs/k8s/controller-manager.conf\n\nkubernetes/server/bin/kubectl config set-context default \\\n--cluster=kubernetes \\\n--user=system:kube-controller-manager \\\n--kubeconfig=certs/k8s/controller-manager.conf\n\nkubernetes/server/bin/kubectl config use-context default \\\n--kubeconfig=certs/k8s/controller-manager.conf\n\n# For scheduler\nkubernetes/server/bin/kubectl config set-cluster kubernetes \\\n--certificate-authority=certs/k8s/k8s-ca.pem \\\n--embed-certs=true \\\n--server=${KUBE_APISERVER} \\\n--kubeconfig=certs/k8s/scheduler.conf\n\nkubernetes/server/bin/kubectl config set-credentials system:kube-scheduler \\\n--client-certificate=certs/k8s/kube-scheduler.pem \\\n--client-key=certs/k8s/kube-scheduler-key.pem \\\n--embed-certs=true \\\n--kubeconfig=certs/k8s/scheduler.conf\n\nkubernetes/server/bin/kubectl config set-context default \\\n--cluster=kubernetes \\\n--user=system:kube-scheduler \\\n--kubeconfig=certs/k8s/scheduler.conf\n\nkubernetes/server/bin/kubectl config use-context default \\\n--kubeconfig=certs/k8s/scheduler.conf\n\n#Check the output in `certs/k8s`\nls certs/k8s\n# admin.conf               kube-admin-csr.json          kube-controller-manager-csr.json  kubelet-etcd-client-csr.json  kube-scheduler.csr\n# bootstrap.conf           kube-admin-key.pem           kube-controller-manager-key.pem   kubelet-etcd-client-key.pem   kube-scheduler-csr.json\n# controller-manager.conf  kube-admin.pem               kube-controller-manager.pem       kubelet-etcd-client.pem       kube-scheduler-key.pem\n# k8s-ca.csr               kube-apiserver.csr           kube-front-proxy-client.csr       kubelet-master.conf           kube-scheduler.pem\n# k8s-ca-csr.json          kube-apiserver-csr.json      kube-front-proxy-client-csr.json  kubelet-master.csr            sa.key\n# k8s-ca-key.pem           kube-apiserver-key.pem       kube-front-proxy-client-key.pem   kubelet-master-csr.json       sa.pub\n# k8s-ca.pem               kube-apiserver.pem           kube-front-proxy-client.pem       kubelet-master-key.pem        scheduler.conf\n# kube-admin.csr           kube-controller-manager.csr  kubelet-etcd-client.csr           kubelet-master.pem            token.csv\n```\n\n- Distribute certificate and kubeconfig file to master and nodes\n\n```bash\n\n#For non-master node, without kubeconfig file,only copy bootstrap.conf\nfor node in ${K8S_NODES} ;do\n  echo Copy node certificate file to $node\n  ssh ${node} mkdir -p /etc/kubernetes/pki\n  for files in \\\n    certs/etcd/etcd-ca.pem \\\n    certs/k8s/kubelet-etcd-client*.pem \\\n    certs/k8s/k8s-ca*.pem ;\\\n  do\n    scp $files ${node}:/etc/kubernetes/pki\n  done\n  scp certs/k8s/bootstrap.conf ${node}:/etc/kubernetes\ndone\n\n#For master node,which has --kubeconfig file copied\necho Copy node certificate file to ${KUBE_MASTER_SERV_01}[master]\nfor files in \\\n  certs/k8s/*.pem \\\n  certs/k8s/sa.* \\\n  certs/etcd/etcd-ca.pem ; \\\ndo\n  scp $files ${KUBE_MASTER_SERV_01}:/etc/kubernetes/pki\ndone\nscp certs/k8s/*.conf ${KUBE_MASTER_SERV_01}:/etc/kubernetes\nscp certs/k8s/token.csv ${KUBE_MASTER_SERV_01}:/etc/kubernetes\n\n```\n\n## 10. Setup `apiserver` `controller-manager` `scheduler` static pods yaml configs\n\n```bash\ncd\nmkdir kubernetes/manifests -p\n```\n\n- For `apiserver` pod\n\n```bash\ncat > kubernetes/manifests/kube-apiserver.yaml << EOF\napiVersion: v1\nkind: Pod\nmetadata:\n  annotations:\n    scheduler.alpha.kubernetes.io/critical-pod: \"\"\n  labels:\n    component: kube-apiserver\n    tier: control-plane\n  name: kube-apiserver\n  namespace: kube-system\nspec:\n  hostNetwork: true\n  containers :\n  - name: kube-apiserver\n    image: ${PRIVATE_REGISTRY}/kube-apiserver:${K8S_BASE_VER}\n    command:\n      - kube-apiserver\n      - --v=0\n      - --logtostderr=true\n      - --allow-privileged=true\n      - --bind-address=0.0.0.0\n      - --secure-port=${KUBE_APISERVER_PORT}\n      - --insecure-port=0\n      - --advertise-address=${KUBE_APISERVER_IP_01}\n      - --service-cluster-ip-range=${KUBE_CLUSTER_SERVICE_CIDR}\n      - --etcd-servers=https://${ETCD1_IP}:${ETCD_CLIENT_PORT},https://${ETCD2_IP}:${ETCD_CLIENT_PORT},https://${ETCD3_IP}:${ETCD_CLIENT_PORT}\n      - --etcd-cafile=/etc/kubernetes/pki/etcd-ca.pem\n      - --etcd-certfile=/etc/kubernetes/pki/kubelet-etcd-client.pem\n      - --etcd-keyfile=/etc/kubernetes/pki/kubelet-etcd-client-key.pem\n      - --client-ca-file=/etc/kubernetes/pki/k8s-ca.pem\n      - --tls-cert-file=/etc/kubernetes/pki/kube-apiserver.pem\n      - --tls-private-key-file=/etc/kubernetes/pki/kube-apiserver-key.pem\n      - --kubelet-client-certificate=/etc/kubernetes/pki/kube-apiserver.pem\n      - --kubelet-client-key=/etc/kubernetes/pki/kube-apiserver-key.pem\n      - --service-account-key-file=/etc/kubernetes/pki/sa.pub\n      - --token-auth-file=/etc/kubernetes/token.csv\n      - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname\n      - --admission-control=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota\n      - --authorization-mode=Node,RBAC\n      - --enable-bootstrap-token-auth=true\n      - --requestheader-client-ca-file=/etc/kubernetes/pki/k8s-ca.pem\n      - --proxy-client-cert-file=/etc/kubernetes/pki/kube-front-proxy-client.pem\n      - --proxy-client-key-file=/etc/kubernetes/pki/kube-front-proxy-client-key.pem\n      #- --requestheader-allowed-names=aggregator\n      - --requestheader-group-headers=X-Remote-Group\n      - --requestheader-extra-headers-prefix=X-Remote-Extra-\n      - --requestheader-username-headers=X-Remote-User\n      - --event-ttl=1h\n    livenessProbe:\n      failureThreshold: 8\n      httpGet:\n        host: 127.0.0.1\n        path: /healthz\n        port: ${KUBE_APISERVER_PORT}\n        scheme: HTTPS\n      initialDelaySeconds: 15\n      timeoutSeconds: 15\n    resources:\n      requests:\n        cpu: 250m\n    volumeMounts:\n    - mountPath: /var/log/kubernetes\n      name: k8s-audit-log\n    - mountPath: /etc/kubernetes/pki\n      name: k8s-certs\n      readOnly: true\n    - mountPath: /etc/kubernetes/token.csv\n      name: token-csv\n      readOnly: true\n  volumes:\n  - hostPath:\n      path: /var/log/kubernetes\n      type: DirectoryOrCreate\n    name: k8s-audit-log\n  - hostPath:\n      path: /etc/kubernetes/pki\n      type: DirectoryOrCreate\n    name: k8s-certs\n  - hostPath:\n      path: /etc/kubernetes/token.csv\n      type: FileOrCreate\n    name: token-csv\nEOF\n```\n- For `controller manager`\n\n```bash\ncat > kubernetes/manifests/kube-controller-manager.yaml << EOF\napiVersion: v1\nkind: Pod\nmetadata:\n  annotations:\n    scheduler.alpha.kubernetes.io/critical-pod: \"\"\n  labels:\n    component: kube-controller-manager\n    tier: control-plane\n  name: kube-controller-manager\n  namespace: kube-system\nspec:\n  hostNetwork: true\n  containers:\n  - name: kube-controller-manager\n    image: ${PRIVATE_REGISTRY}/kube-controller-manager:${K8S_BASE_VER}\n    command:\n      - kube-controller-manager\n      - --v=0\n      - --logtostderr=true\n      - --address=127.0.0.1\n      - --root-ca-file=/etc/kubernetes/pki/k8s-ca.pem\n      - --cluster-signing-cert-file=/etc/kubernetes/pki/k8s-ca.pem\n      - --cluster-signing-key-file=/etc/kubernetes/pki/k8s-ca-key.pem\n      - --service-account-private-key-file=/etc/kubernetes/pki/sa.key\n      - --kubeconfig=/etc/kubernetes/controller-manager.conf\n      - --leader-elect=true\n      - --use-service-account-credentials=true\n      - --node-monitor-grace-period=40s\n      - --node-monitor-period=5s\n      - --pod-eviction-timeout=2m0s\n      - --controllers=*,bootstrapsigner,tokencleaner\n      - --allocate-node-cidrs=true\n      - --cluster-cidr=${KUBE_CLUSTER_POD_CIDR}\n      - --node-cidr-mask-size=24\n    livenessProbe:\n      failureThreshold: 8\n      httpGet:\n        host: 127.0.0.1\n        path: /healthz\n        port: 10252\n        scheme: HTTP\n      initialDelaySeconds: 15\n      timeoutSeconds: 15\n    resources:\n      requests:\n        cpu: 200m\n    volumeMounts:\n    - mountPath: /etc/kubernetes/pki\n      name: k8s-certs\n      readOnly: true\n    - mountPath: /etc/ssl/certs\n      name: ca-certs\n      readOnly: true\n    - mountPath: /etc/kubernetes/controller-manager.conf\n      name: kubeconfig\n      readOnly: true\n    - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec\n      name: flexvolume-dir\n  volumes:\n  - hostPath:\n      path: /etc/kubernetes/pki\n      type: DirectoryOrCreate\n    name: k8s-certs\n  - hostPath:\n      path: /etc/ssl/certs\n      type: DirectoryOrCreate\n    name: ca-certs\n  - hostPath:\n      path: /etc/kubernetes/controller-manager.conf\n      type: FileOrCreate\n    name: kubeconfig\n  - hostPath:\n      path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec\n      type: DirectoryOrCreate\n    name: flexvolume-dir\nEOF\n```\n\n- For `scheduler`\n\n```bash\ncat > kubernetes/manifests/kube-scheduler.yaml << EOF\napiVersion: v1\nkind: Pod\nmetadata:\n  annotations:\n    scheduler.alpha.kubernetes.io/critical-pod: \"\"\n  labels:\n    component: kube-scheduler\n    tier: control-plane\n  name: kube-scheduler\n  namespace: kube-system\nspec:\n  hostNetwork: true\n  containers:\n  - name: kube-scheduler\n    image: ${PRIVATE_REGISTRY}/kube-scheduler:${K8S_BASE_VER}\n    command:\n      - kube-scheduler\n      - --v=0\n      - --logtostderr=true\n      - --address=127.0.0.1\n      - --leader-elect=true\n      - --kubeconfig=/etc/kubernetes/scheduler.conf\n    livenessProbe:\n      failureThreshold: 8\n      httpGet:\n        host: 127.0.0.1\n        path: /healthz\n        port: 10251\n        scheme: HTTP\n      initialDelaySeconds: 15\n      timeoutSeconds: 15\n    resources:\n      requests:\n        cpu: 100m\n    volumeMounts:\n    - mountPath: /etc/kubernetes/pki\n      name: k8s-certs\n      readOnly: true\n    - mountPath: /etc/kubernetes/scheduler.conf\n      name: kubeconfig\n      readOnly: true\n  volumes:\n  - hostPath:\n      path: /etc/kubernetes/pki\n      type: DirectoryOrCreate\n    name: k8s-certs\n  - hostPath:\n      path: /etc/kubernetes/scheduler.conf\n      type: FileOrCreate\n    name: kubeconfig\nEOF\n```\n- Distribute to master node\n\n```bash\nssh ${KUBE_MASTER_SERV_01} mkdir -p /etc/kubernetes/manifests\nscp kubernetes/manifests/*.yaml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/manifests\n```\n\n## 11. Setup and boot the `kubelet` & **cluster** with systemd service\n\n```bash\ncd\nmkdir kubernetes/systemd/ -p\n\ncat > kubernetes/systemd/kubelet.service << EOF\n[Unit]\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=http://kubernetes.io/docs/\n\n[Service]\nExecStart=/usr/local/bin/kubelet\nRestart=on-failure\nStartLimitInterval=0\nRestartSec=10\n\n[Install]\nWantedBy=multi-user.target\nEOF\n\ncat > kubernetes/systemd/10-kubelet.conf << EOF\n[Service]\nEnvironment=\"KUBELET_KUBECONFIG_ARGS=--address=0.0.0.0 --port=10250 --kubeconfig=/etc/kubernetes/kubelet-master.conf --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.conf\"\nEnvironment=\"KUBE_LOGTOSTDERR=--logtostderr=true --v=0\"\nEnvironment=\"KUBELET_SYSTEM_PODS_ARGS=--pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --anonymous-auth=false\"\nEnvironment=\"KUBELET_POD_CONTAINER=--pod-infra-container-image=${PRIVATE_REGISTRY}/${K8S_PAUSE_IMG_NAME}:${K8S_PAUSE_VER}\"\nEnvironment=\"KUBELET_NETWORK_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin\"\nEnvironment=\"KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_IP} --cluster-domain=cluster.local\"\nEnvironment=\"KUBELET_AUTHZ_ARGS=--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/k8s-ca.pem\"\nEnvironment=\"KUBELET_CADVISOR_ARGS=--cadvisor-port=0\"\nEnvironment=\"KUBELET_CERTIFICATE_ARGS=--rotate-certificates=true --cert-dir=/var/lib/kubelet/pki\"\nEnvironment=\"KUBELET_EXTRA_ARGS=--fail-swap-on=false --serialize-image-pulls=false\"\nEnvironment=\"KUBE_NODE_LABEL=--node-labels=node-role.kubernetes.io/master=true\"\n#kubelet cgroup-driver MUST be consistence with docker daemon\nEnvironment=\"KUBELET_CGROUP=--cgroup-driver=systemd\"\nExecStart=\nExecStart=/usr/local/bin/kubelet \\$KUBELET_KUBECONFIG_ARGS \\$KUBE_LOGTOSTDERR \\$KUBELET_POD_CONTAINER \\$KUBELET_SYSTEM_PODS_ARGS \\$KUBELET_NETWORK_ARGS \\$KUBELET_DNS_ARGS \\$KUBELET_AUTHZ_ARGS \\$KUBELET_EXTRA_ARGS \\$KUBE_NODE_LABEL \\$KUBELET_CGROUP\nEOF\n\nfor node in ${K8S_NODES} ;do\n  echo Prepare for $node\n  ssh ${node} mkdir -p /etc/systemd/system/kubelet.service.d\n  scp kubernetes/systemd/10-kubelet.conf ${node}:/etc/systemd/system/kubelet.service.d/\n  scp kubernetes/systemd/kubelet.service ${node}:/lib/systemd/system/kubelet.service\n  ssh ${node} \"systemctl stop kubelet && systemctl disable kubelet\"\n  ssh ${node} \"systemctl enable kubelet && systemctl start kubelet\"\ndone\n```\n\n## 12. Setup admin cli environment and check cluster status\n- Setup the admin cli environment with `amdin.con` file\n\n```bash\nssh ${KUBE_MASTER_SERV_01} mkdir ~/.kube\nssh ${KUBE_MASTER_SERV_01} cp /etc/kubernetes/admin.conf ~/.kube/config\nssh ${KUBE_MASTER_SERV_01} kubectl get cs\nssh ${KUBE_MASTER_SERV_01} kubectl get node\nssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po\n```\n- The run out result should be like this:\n\n```bash\nssh ${KUBE_MASTER_SERV_01} kubectl get cs\n# NAME                 STATUS    MESSAGE              ERROR\n# controller-manager   Healthy   ok\n# scheduler            Healthy   ok\n# etcd-2               Healthy   {\"health\": \"true\"}\n# etcd-0               Healthy   {\"health\": \"true\"}\n# etcd-1               Healthy   {\"health\": \"true\"}\nssh ${KUBE_MASTER_SERV_01} kubectl get node\n# NAME      STATUS     ROLES     AGE       VERSION\n# jin       NotReady   master    1m        v1.9.0\nssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po\n# NAME                          READY     STATUS    RESTARTS   AGE\n# kube-apiserver-jin            1/1       Running   0          40s\n# kube-controller-manager-jin   1/1       Running   0          33s\n# kube-scheduler-jin            1/1       Running   0          38s\n```\n\n## 13. Authorize other K8s nodes joining the cluster\n- Add TLS Bootstrapping ClusterRoleBinding to cluster\n\n```bash\nssh ${KUBE_MASTER_SERV_01} kubectl create \\\n  clusterrolebinding kubelet-bootstrap \\\n  --clusterrole=system:node-bootstrapper \\\n  --user=kubelet-bootstrap\n```\n\n- Check pending node certificate CSR requests\n\n```bash\nssh ${KUBE_MASTER_SERV_01} kubectl get csr\n# NAME                                                   AGE       REQUESTOR           CONDITION\n# node-csr-gO_HGaj_cJdNzjx3AxjmMtKBNfiBRUdIfBn0HdDyWtM   8s        kubelet-bootstrap   Pending\n# node-csr-nrYBLZlNzplcs5pW81BvKTzzenqzXCSQ1IddYFs0BLY   8s        kubelet-bootstrap   Pending\n# node-csr-t6TpFDn-uaou9RWRBs0cAXd9V2RIDgretORPWkM0--E   7s        kubelet-bootstrap   Pending\n```\n\n- Authorize node CSR requests\n\n```bash\nssh ${KUBE_MASTER_SERV_01} \"kubectl get csr | awk '/Pending/ {print \\$1}' | xargs kubectl certificate approve\"\n# certificatesigningrequest \"node-csr-gO_HGaj_cJdNzjx3AxjmMtKBNfiBRUdIfBn0HdDyWtM\" approved\n# certificatesigningrequest \"node-csr-nrYBLZlNzplcs5pW81BvKTzzenqzXCSQ1IddYFs0BLY\" approved\n# certificatesigningrequest \"node-csr-t6TpFDn-uaou9RWRBs0cAXd9V2RIDgretORPWkM0--E\" approved\n\nssh ${KUBE_MASTER_SERV_01} kubectl get nodes\n# NAME      STATUS     ROLES     AGE       VERSION\n# huo       NotReady   master    34s       v1.9.0\n# jin       NotReady   master    3m        v1.9.0\n# mu        NotReady   master    33s       v1.9.0\n# shui      NotReady   master    34s       v1.9.0\n```\n\n## 14. Setup `kube-proxy` addon\nWhich shall be deployed ahead of `Calico` ,otherwise the `Calico` components would stuck in creating container status.\n\n```bash\ncd\nmkdir -p kubernetes/addons\n\n#Generate kube proxy certificate\ncat <<EOF > certs/k8s/kube-proxy-csr.json\n{\n  \"CN\":\"system:kube-proxy\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  },\n  \"names\":[\n    {\n      \"O\":\"system:kube-proxy\"\n    }\n  ]\n}\nEOF\n\ncfssl gencert -ca=certs/k8s/k8s-ca.pem \\\n  -ca-key=certs/k8s/k8s-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/k8s/kube-proxy-csr.json | cfssljson -bare certs/k8s/kube-proxy\n\n#Generate kubeconfig for proxy\nkubernetes/server/bin/kubectl config set-cluster kubernetes \\\n--certificate-authority=certs/k8s/k8s-ca.pem \\\n--embed-certs=true \\\n--server=${KUBE_APISERVER} \\\n--kubeconfig=certs/k8s/kube-proxy.conf\n\nkubernetes/server/bin/kubectl config set-credentials system:kube-proxy \\\n--client-certificate=certs/k8s/kube-scheduler.pem \\\n--client-key=certs/k8s/kube-scheduler-key.pem \\\n--embed-certs=true \\\n--kubeconfig=certs/k8s/kube-proxy.conf\n\nkubernetes/server/bin/kubectl config set-context default \\\n--cluster=kubernetes \\\n--user=system:kube-proxy \\\n--kubeconfig=certs/k8s/kube-proxy.conf\n\nkubernetes/server/bin/kubectl config use-context default \\\n--kubeconfig=certs/k8s/kube-proxy.conf\n\n#Generate kube-proxy yaml\ncat > kubernetes/addons/kube-proxy.yml <<EOF\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: kube-proxy\n  labels:\n    k8s-app: kube-proxy\n    kubernetes.io/cluster-service: \"true\"\n    addonmanager.kubernetes.io/mode: Reconcile\n  namespace: kube-system\n---\napiVersion: extensions/v1beta1\nkind: DaemonSet\nmetadata:\n  name: kube-proxy\n  labels:\n    k8s-app: kube-proxy\n    kubernetes.io/cluster-service: \"true\"\n    addonmanager.kubernetes.io/mode: Reconcile\n  namespace: kube-system\nspec:\n  selector:\n    matchLabels:\n      k8s-app: kube-proxy\n  templateGeneration: 1\n  updateStrategy:\n    rollingUpdate:\n      maxUnavailable: 1\n    type: RollingUpdate\n  template:\n    metadata:\n      labels:\n        k8s-app: kube-proxy\n      annotations:\n        scheduler.alpha.kubernetes.io/critical-pod: ''\n    spec:\n      serviceAccountName: kube-proxy\n      hostNetwork: true\n      containers:\n      - name: kube-proxy\n        image: ${PRIVATE_REGISTRY}/${K8S_PROXY_IMG_NAME}:${K8S_PROXY_VER}\n        command:\n        - kube-proxy\n        - --v=0\n        - --logtostderr=true\n        - --kubeconfig=/run/kube-proxy.conf\n        - --cluster-cidr=${KUBE_CLUSTER_POD_CIDR}\n        - --proxy-mode=iptables\n        imagePullPolicy: IfNotPresent\n        securityContext:\n          privileged: true\n        volumeMounts:\n        - mountPath: /run/kube-proxy.conf\n          name: kubeconfig\n          readOnly: true\n        - mountPath: /etc/kubernetes/pki\n          name: k8s-certs\n          readOnly: true\n      dnsPolicy: ClusterFirst\n      restartPolicy: Always\n      terminationGracePeriodSeconds: 30\n      volumes:\n      - hostPath:\n          path: /etc/kubernetes/kube-proxy.conf\n          type: FileOrCreate\n        name: kubeconfig\n      - hostPath:\n          path: /etc/kubernetes/pki\n          type: DirectoryOrCreate\n        name: k8s-certs\nEOF\n\n#Distribute certs and config file to nodes\nfor node in ${K8S_NODES} ; do\n  scp certs/k8s/kube-proxy*.pem ${node}:/etc/kubernetes/pki\n  scp certs/k8s/kube-proxy.conf ${node}:/etc/kubernetes/\ndone\nssh ${KUBE_MASTER_SERV_01} \"mkdir -p /etc/kubernetes/addons\"\nscp kubernetes/addons/kube-proxy.yml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/addons\n\n#Apply yaml to cluster\nssh ${KUBE_MASTER_SERV_01} kubectl apply -f /etc/kubernetes/addons/kube-proxy.yml\n\n#Check status\nssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po -l k8s-app=kube-proxy\n# NAME               READY     STATUS    RESTARTS   AGE\n# kube-proxy-drfbj   1/1       Running   0          45s\n# kube-proxy-lx6sx   1/1       Running   0          45s\n# kube-proxy-m44b8   1/1       Running   0          45s\n# kube-proxy-nb2sw   1/1       Running   0          44s\n\n```\n\n## 15. `Calico` Standard Hosted Install\n- Generate certificate for calico-etcd-client\n\n```bash\ncd\nmkdir certs/calico -p\ncat <<EOF > certs/calico/calico-etcd-client-csr.json\n{\n  \"CN\":\"calico-nodes\",\n  \"key\":{\n    \"algo\":\"rsa\",\n    \"size\":2048\n  }\n}\nEOF\n\ncfssl gencert -ca=certs/etcd/etcd-ca.pem \\\n  -ca-key=certs/etcd/etcd-ca-key.pem \\\n  -config=certs/ca-config.json \\\n  -profile=default \\\n  certs/calico/calico-etcd-client-csr.json | cfssljson -bare certs/calico/calico-etcd-client\n\n```\n- Modify preset yaml pattern\nSetup Calico network in IPIP `OFF` mode,the pattern is fetched from [Standard Hosted Install](https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/hosted) - [calico.yaml](https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/calico.yaml)\n\n```bash\nexport ETCD_CA_BASE64=`cat ${HOME}/certs/etcd/etcd-ca.pem|base64 -w 0`\nexport ETCD_CERT_BASE64=`cat ${HOME}/certs/calico/calico-etcd-client.pem|base64 -w 0`\nexport ETCD_KEY_BASE64=`cat ${HOME}/certs/calico/calico-etcd-client-key.pem|base64 -w 0`\nsed \"s#http://127.0.0.1:2379#$ETCD_ENDPOINTS#\" \\\nall-in-one/configs/calico-deploy.yaml.conf |\\\nsed 's/etcd_ca: \"\"   #/etcd_ca: /g' |\\\nsed 's/etcd_cert: \"\" #/etcd_cert: /g' |\\\nsed 's/etcd_key: \"\"  #/etcd_key: /g' |\\\nsed \"s/  # etcd-key: null/  etcd-key: $ETCD_KEY_BASE64/\" |\\\nsed \"s/  # etcd-cert: null/  etcd-cert: $ETCD_CERT_BASE64/\" |\\\nsed \"s/  # etcd-ca: null/  etcd-ca: $ETCD_CA_BASE64/\" |\\\nsed \"s#quay.io/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}#${PRIVATE_REGISTRY}/calico/${CALICO_NODE_IMAGE_NAME}:${CALICO_NODE_VER}#g\" |\\\nsed \"s#192.168.0.0/16#${KUBE_CLUSTER_POD_CIDR}#\" |\\\nsed \"s#quay.io/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}#${PRIVATE_REGISTRY}/calico/${CALICO_CNI_IMAGE_NAME}:${CALICO_CNI_VER}#g\" |\\\nsed \"s#quay.io/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}#${PRIVATE_REGISTRY}/calico/${CALICO_CONTROLLER_IMG_NAME}:${CALICO_CONTROLLER_VER}#g\" |\\\nsed \"/CALICO_IPV4POOL_IPIP/{n;d}\" |\\\nsed '/CALICO_IPV4POOL_IPIP/a\\              value: \"off\"' > \\\nall-in-one/configs/calico-deploy.yaml\n\n#Distribute config file to master node\nscp all-in-one/configs/calico-deploy.yaml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/addons\nscp all-in-one/configs/calico-rbac.yaml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/addons\nssh ${KUBE_MASTER_SERV_01} kubectl apply -f /etc/kubernetes/addons/calico-rbac.yaml\nssh ${KUBE_MASTER_SERV_01} kubectl apply -f /etc/kubernetes/addons/calico-deploy.yaml\n\n#Check the calico pod status\nssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po\n# NAME                                       READY     STATUS    RESTARTS   AGE\n# calico-kube-controllers-54747bbfbf-rx48m   1/1       Running   0          2m\n# calico-node-2h6r8                          2/2       Running   0          2m\n# calico-node-gfwzf                          2/2       Running   0          2m\n# calico-node-gvr7v                          2/2       Running   0          2m\n# calico-node-lf4zp                          2/2       Running   0          2m\n# kube-apiserver-jin                         1/1       Running   0          16m\n# kube-controller-manager-jin                1/1       Running   0          16m\n# kube-proxy-drfbj                           1/1       Running   0          12m\n# kube-proxy-lx6sx                           1/1       Running   0          12m\n# kube-proxy-m44b8                           1/1       Running   0          12m\n# kube-proxy-nb2sw                           1/1       Running   0          12m\n# kube-scheduler-jin                         1/1       Running   0          16m\n```\n\n## 16. Setup `kube-dns` addon\n\n```bash\ncat > kubernetes/addons/kube-dns.yml << EOF\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n  name: kube-dns\n  labels:\n    k8s-app: kube-dns\n    kubernetes.io/cluster-service: \"true\"\n    addonmanager.kubernetes.io/mode: Reconcile\n  namespace: kube-system\n---\napiVersion: v1\nkind: Service\nmetadata:\n  name: kube-dns\n  namespace: kube-system\n  labels:\n    k8s-app: kube-dns\n    kubernetes.io/cluster-service: \"true\"\n    addonmanager.kubernetes.io/mode: Reconcile\nspec:\n  selector:\n    k8s-app: kube-dns\n  clusterIP: ${CLUSTER_DNS_IP}\n  ports:\n  - name: dns\n    port: 53\n    protocol: UDP\n  - name: dns-tcp\n    port: 53\n    protocol: TCP\n---\napiVersion: extensions/v1beta1\nkind: Deployment\nmetadata:\n  name: kube-dns\n  namespace: kube-system\n  labels:\n    k8s-app: kube-dns\n    kubernetes.io/cluster-service: \"true\"\n    addonmanager.kubernetes.io/mode: Reconcile\nspec:\n  strategy:\n    rollingUpdate:\n      maxSurge: 10%\n      maxUnavailable: 0\n  selector:\n    matchLabels:\n      k8s-app: kube-dns\n  template:\n    metadata:\n      labels:\n        k8s-app: kube-dns\n      annotations:\n        scheduler.alpha.kubernetes.io/critical-pod: ''\n    spec:\n      dnsPolicy: Default\n      serviceAccountName: kube-dns\n      tolerations:\n      - key: \"CriticalAddonsOnly\"\n        operator: \"Exists\"\n      - key: node-role.kubernetes.io/master\n        effect: NoSchedule\n      volumes:\n      - name: kube-dns-config\n        configMap:\n          name: kube-dns\n          optional: true\n      containers:\n      - name: kubedns\n        image: ${PRIVATE_REGISTRY}/${K8S_KUBE_DNS_IMG_NAME}:${K8S_DNS_VER}\n        resources:\n          limits:\n            memory: 170Mi\n          requests:\n            cpu: 100m\n            memory: 70Mi\n        livenessProbe:\n          httpGet:\n            path: /healthcheck/kubedns\n            port: 10054\n            scheme: HTTP\n          initialDelaySeconds: 60\n          timeoutSeconds: 5\n          successThreshold: 1\n          failureThreshold: 5\n        readinessProbe:\n          httpGet:\n            path: /readiness\n            port: 8081\n            scheme: HTTP\n          initialDelaySeconds: 3\n          timeoutSeconds: 5\n        args:\n        - \"--domain=cluster.local\"\n        - --dns-port=10053\n        - --v=2\n        env:\n        - name: PROMETHEUS_PORT\n          value: \"10055\"\n        ports:\n        - containerPort: 10053\n          name: dns-local\n          protocol: UDP\n        - containerPort: 10053\n          name: dns-tcp-local\n          protocol: TCP\n        - containerPort: 10055\n          name: metrics\n          protocol: TCP\n        volumeMounts:\n        - name: kube-dns-config\n          mountPath: /kube-dns-config\n      - name: dnsmasq\n        image: ${PRIVATE_REGISTRY}/${K8S_DNSMASQ_IMG_NAME}:${K8S_DNS_VER}\n        livenessProbe:\n          httpGet:\n            path: /healthcheck/dnsmasq\n            port: 10054\n            scheme: HTTP\n          initialDelaySeconds: 60\n          timeoutSeconds: 5\n          successThreshold: 1\n          failureThreshold: 5\n        args:\n        - \"-v=2\"\n        - \"-logtostderr\"\n        - \"-configDir=/etc/k8s/dns/dnsmasq-nanny\"\n        - \"-restartDnsmasq=true\"\n        - \"--\"\n        - \"-k\"\n        - \"--cache-size=1000\"\n        - \"--log-facility=-\"\n        - \"--server=/cluster.local/127.0.0.1#10053\"\n        - \"--server=/in-addr.arpa/127.0.0.1#10053\"\n        - \"--server=/ip6.arpa/127.0.0.1#10053\"\n        ports:\n        - containerPort: 53\n          name: dns\n          protocol: UDP\n        - containerPort: 53\n          name: dns-tcp\n          protocol: TCP\n        resources:\n          requests:\n            cpu: 150m\n            memory: 20Mi\n        volumeMounts:\n        - name: kube-dns-config\n          mountPath: /etc/k8s/dns/dnsmasq-nanny\n      - name: sidecar\n        image: ${PRIVATE_REGISTRY}/${K8S_SIDECAR_IMG_NAME}:${K8S_DNS_VER}\n        livenessProbe:\n          httpGet:\n            path: /metrics\n            port: 10054\n            scheme: HTTP\n          initialDelaySeconds: 60\n          timeoutSeconds: 5\n          successThreshold: 1\n          failureThreshold: 5\n        args:\n        - \"--v=2\"\n        - \"--logtostderr\"\n        - \"--probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A\"\n        - \"--probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A\"\n        ports:\n        - containerPort: 10054\n          name: metrics\n          protocol: TCP\n        resources:\n          requests:\n            memory: 20Mi\n            cpu: 10m\nEOF\n\n#Distribute certs and config file to nodes\nssh ${KUBE_MASTER_SERV_01} mkdir -p /etc/kubernetes/addons\nscp kubernetes/addons/kube-dns.yml ${KUBE_MASTER_SERV_01}:/etc/kubernetes/addons\n\n#Apply yaml to cluster\nssh ${KUBE_MASTER_SERV_01} kubectl apply -f /etc/kubernetes/addons/kube-dns.yml\n\n#Check DNS status with cluster service IP\nssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get po -l k8s-app=kube-dns -o wide\n# NAME                       READY     STATUS    RESTARTS   AGE       IP             NODE\n# kube-dns-c84c85bb4-svv9v   3/3       Running   0          2m        172.17.95.65   mu\n```\n\n## 17. Check the whole cluster status\nThat's all for the cluster deployments, now you can check the overall status in the cluster\n\n```bash\nssh ${KUBE_MASTER_SERV_01} kubectl -n kube-system get all\n# NAME             DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE\n# ds/calico-node   4         4         4         4            4           <none>          5m\n# ds/kube-proxy    4         4         4         4            4           <none>          16m\n#\n# NAME                              DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE\n# deploy/calico-kube-controllers    1         1         1            1           5m\n# deploy/calico-policy-controller   0         0         0            0           5m\n# deploy/kube-dns                   1         1         1            1           1m\n#\n# NAME                                     DESIRED   CURRENT   READY     AGE\n# rs/calico-kube-controllers-54747bbfbf    1         1         1         5m\n# rs/calico-policy-controller-6d964cc58f   0         0         0         5m\n# rs/kube-dns-c84c85bb4                    1         1         1         1m\n#\n# NAME             DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE\n# ds/calico-node   4         4         4         4            4           <none>          5m\n# ds/kube-proxy    4         4         4         4            4           <none>          16m\n#\n# NAME                              DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE\n# deploy/calico-kube-controllers    1         1         1            1           5m\n# deploy/calico-policy-controller   0         0         0            0           5m\n# deploy/kube-dns                   1         1         1            1           1m\n#\n# NAME                                     DESIRED   CURRENT   READY     AGE\n# rs/calico-kube-controllers-54747bbfbf    1         1         1         5m\n# rs/calico-policy-controller-6d964cc58f   0         0         0         5m\n# rs/kube-dns-c84c85bb4                    1         1         1         1m\n#\n# NAME                                          READY     STATUS    RESTARTS   AGE\n# po/calico-kube-controllers-54747bbfbf-rx48m   1/1       Running   0          5m\n# po/calico-node-2h6r8                          2/2       Running   0          5m\n# po/calico-node-gfwzf                          2/2       Running   0          5m\n# po/calico-node-gvr7v                          2/2       Running   0          5m\n# po/calico-node-lf4zp                          2/2       Running   0          5m\n# po/kube-apiserver-jin                         1/1       Running   0          19m\n# po/kube-controller-manager-jin                1/1       Running   0          19m\n# po/kube-dns-c84c85bb4-svv9v                   3/3       Running   0          1m\n# po/kube-proxy-drfbj                           1/1       Running   0          16m\n# po/kube-proxy-lx6sx                           1/1       Running   0          16m\n# po/kube-proxy-m44b8                           1/1       Running   0          16m\n# po/kube-proxy-nb2sw                           1/1       Running   0          15m\n# po/kube-scheduler-jin                         1/1       Running   0          19m\n#\n# NAME           TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)         AGE\n# svc/kube-dns   ClusterIP   172.16.0.10   <none>        53/UDP,53/TCP   1m\n```\n\n### References\n- [Standard Hosted Install](https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/hosted)\n\n- [Kubernetes 1.8.x 全手动安装教程](https://www.kubernetes.org.cn/3096.html)",
      "json_metadata": "{\"tags\":[\"kubernetes\",\"calico\",\"k8s\",\"cn\"],\"links\":[\"https://kubernetes.io/\",\"https://docs.projectcalico.org/v2.6/introduction/\",\"https://docs.docker.com/registry/deploying/\",\"https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/hosted\",\"https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/calico.yaml\",\"https://www.kubernetes.org.cn/3096.html\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}",
      "parent_author": "",
      "parent_permlink": "kubernetes",
      "permlink": "deploy-kubernetes-and-calico-network-in-air-gapped-environment-within-20-minutes",
      "title": "Deploy kubernetes & Calico Network in air-gapped environment within 20 minutes"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T23:58:27",
  "trx_id": "f0825aff1e664fbf877b9ed63547cb54bcbd4d72",
  "trx_in_block": 0,
  "virtual_op": 0
}
highland0971published a new post: deploy-a-private-docker-registry-in-air-gapped-environment
2018/02/01 15:40:54
19,492,368|d287a01
authorhighland0971
body# Prepare required Docker environments ## 1. Pre-defined environment - Internet availabe host: **localhost** - Air-gaped host: **master01.airgapped.org** ## 2. Pre-request - You have to use the same linux distribution with your air-gaped host, which is **master01.airgapped.org**, in this artical we adopt **CentOS 7** as host system. - You'd have to obtain a **PURE** linux environment which **HAVE** Internet access without any post-installed packages as was installed in **master01.airgapped.org**. Otherwise you may encounter failed dependencies for docker in the **master01.airgapped.org**. - We assume that you have a **PURE** Internet access availabe host named **localhost**. ### 2.1 Download and install the required packages for Docker - On the **localhost** , run following commands: ```bash mkdir ~/rpms sudo yum update && yum install -y --downloadonly --downloaddir=${HOME}/rpms docker ssh [email protected] mkdir packages #You can verify the outcome by type ssh [email protected] "pwd && ls" scp ${HOME}/rpms/* [email protected]:~/packages ``` - On the **master01.airgapped.org** , run following commands to install Docker environment ```bash cd ~/packages sudo rpm -ivh * ``` ​If you encoutner missing depandency error like > warning: audit-libs-python-2.7.6-3.el7.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID f4a80eb5: NOKEY > error: Failed dependencies: > libyajl.so.2()(64bit) is needed by oci-systemd-hook-1:0.1.14-1.git1ba44c6.el7.x86_64 > libyajl.so.2()(64bit) is needed by oci-umount-2:2.3.0-1.git51e7c50.el7.x86_64 > libcgroup is needed by policycoreutils-python-2.5-17.1.el7.x86_64 ​On the **localhost** you can run ```bash sudo yum update && yum reinstall -y --downloadonly --downloaddir=${HOME}/rpms missing-packages(like yajl libcgroup) ``` ​And redo the scp command for the missing rpms. ### 2.2 Setup Docker environment and Shoot! - On the **master01.airgapped.org** , following the procedure defined in [Manage Docker as a non-root user](https://docs.docker.com/engine/installation/linux/linux-postinstall/#manage-docker-as-a-non-root-user) to add docker user group. ```bash sudo groupadd docker sudo usermod -aG docker $USER ``` You have to re-login to make current $USER group affect. - On the **master01.airgapped.org** , disable SeLinux in order to allow docker container access the host path file. ```bash sudo setenforce 0 sudo sed 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config > /etc/selinux/config ``` Verify the whether selinux is disabled: ```bash $ getenforce Permissive ``` - On the **master01.airgapped.org** , enable and start the docker deamon. ```bash sudo systemctl enable docker && sudo systemctl start docker ``` - On the **master01.airgapped.org** , check whether the docker's cgroupdriver is set to systemd `--exec-opt native.cgroupdriver=systemd` ```bash cat /usr/lib/systemd/system/docker.service ``` If not: - Edit the `/usr/lib/systemd/system/docker.service` file and add `--exec-opt native.cgroupdriver=systemd` to `/usr/lib/systemd/system/docker.service`'s ExecStart strings. - Restart the docker deamon service with ```bash sudo systemctl reload-deamon docker && sudo systemctl restart docker ``` - On the **master01.airgapped.org** , check whether docker start successfully. ```bash systemctl status docker ``` # Setup private registry service ## 3. Pre-request - You have pulled the newset docker registry image from docker.io and export it to the tar archive. ```bash docker pull registry \ && docker save docker.io/registry:latest -o docker-io.registry.tar ``` - You have to import the registry image to the **master01.airgapped.org** docker image repo cache On **master01.airgapped.org** ```bash docker load -i docker-io.registry.tar ``` ### 3.1 Create HTTPS certificates required for docker private registry This procedure follows the example on [Openssl certificate creation on k8s official document](https://kubernetes.io/docs/concepts/cluster-administration/certificates/#openssl) On **master01.airgapped.org** ： - Prepare cert storage directory ```bash sudo mkdir /usr/lib/certs -p pushd /usr/lib/certs ``` - Generate CA key file ```bash sudo openssl genrsa -out ca.key 2048 ``` - Generate CA certificate file ```bash sudo openssl req -x509 -new -nodes -key ca.key -subj "/CN=${MASTER_IP}" -days 10000 -out ca.crt ``` You can replace `${MASTER_IP}` with your actual or persuade CA server ip address like ```bash sudo openssl req -x509 -new -nodes -key ca.key -subj "/CN=8.8.8.8" -days 10000 -out ca.crt ``` - Generate docker docker registry server key file ```bash sudo openssl genrsa -out private-registry-server.key 2048 ``` - Create a config file for generating a Certificate Signing Request (CSR) for docker registry server ```bash cat > /tmp/private-registry-server.conf <<EOF [ req ] default_bits = 2048 prompt = no default_md = sha256 req_extensions = req_ext distinguished_name = dn [ dn ] C = cn #replace with actual info ST = Tianjin #replace with actual info L = Tianjin #replace with actual info O = Air Gaped Company #replace with actual info OU = Air Gaped team #replace with actual info CN = master01.airgapped.org #replace with actual registry server name [ req_ext ] subjectAltName = @alt_names [ alt_names ] DNS.1 = master01.airgapped.org #replace with alternative dns resolve name DNS.2 = registry.airgapped.org #replace with alternative dns resolve name IP.1 = 192.168.122.163 #replace with actuel ip for the registry server [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=keyEncipherment,dataEncipherment extendedKeyUsage=serverAuth,clientAuth subjectAltName=@alt_names EOF sudo scp /tmp/private-registry-server.conf ./ ``` - Generate the certificate signing request based on the config file for docker registry server ```bash sudo openssl req -new -key private-registry-server.key \ -out private-registry-server.csr \ -config private-registry-server.conf ``` - Sign the registry server certificate with CA cert and docker registry server CSR file ```bash sudo openssl x509 -req -in private-registry-server.csr -CA ca.crt -CAkey ca.key \ -CAcreateserial -out private-registry-server.crt -days 10000 \ -extensions v3_ext -extfile private-registry-server.conf ``` - Verify the final cert for registry server ```bash openssl x509 -noout -text -in private-registry-server.crt ``` ### 3.2 Start private registry - Copy the CA certificate to docker's certs directory to avoid insecur repository error, the directory which certs resides in must have the same hostname with the private registry domain name. ```bash sudo mkdir /etc/docker/certs.d/registry.airgapped.org -p sudo cp /usr/lib/certs/ca.crt /etc/docker/certs.d/registry.airgapped.org ``` - Run registry container with preset certs and port config ```bash sudo mkdir /mnt/docker_images docker run -d --restart=always -v /mnt/docker_images:/var/lib/registry \ -v /usr/lib/certs:/cert \ -e REGISTRY_HTTP_TLS_CERTIFICATE=/cert/private-registry-server.crt \ -e REGISTRY_HTTP_TLS_KEY=/cert/private-registry-server.key \ -e REGISTRY_HTTP_ADDR=0.0.0.0:443 \ -p 443:443 --name local-registry registry ``` - Verify private registry successfully started and listen on assigned port ```bash docker logs local-registry ``` You should see the following output like,with **listening on [::]:443** > time="2017-12-18T12:57:53Z" level=warning msg="No HTTP secret provided - generated random secret. This may cause problems with uploads if multiple registries are behind a load-balancer. To provide a shared secret, fill in http.secret in the configuration file or set the REGISTRY_HTTP_SECRET environment variable." go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2 > time="2017-12-18T12:57:53Z" level=info msg="redis not configured" go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2 > time="2017-12-18T12:57:53Z" level=info msg="Starting upload purge in 37m0s" go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2 > time="2017-12-18T12:57:54Z" level=info msg="using inmemory blob descriptor cache" go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2 > time="2017-12-18T12:57:54Z" level=info **msg="listening on [::]:443, tls"** go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2 - Append **registry.airgapped.org** domain to /etc/hosts, as dedicate domain for registry service ```bash sudo sh -c 'echo "192.168.122.163 registry.airgapped.org" >> /etc/hosts' ``` If every thing is ok, you shall see the result by verifing the registry v2 API ```bash curl --cacert /usr/lib/certs/ca.crt https://registry.airgapped.org/v2/_catalog {"repositories":[]} ``` ## 3.3 Push your first image to private registry and verify On **master01.airgapped.org**, At this point, you have: - one available private registry which is host on **master01.airgapped.org** and with a dedicate domain name **registry.airgapped.org** with TLS enabled, listening on port 443. - one local image named **docker.io/registry:latest** cached on **master01.airgapped.org** , as: ```bash $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE docker.io/registry latest 177391bcf802 2 weeks ago 33.26 MB ``` Now you can verify the service by the following scripts: 3.3.1. **Tag cached image with new registry path** which leads to **registry.airgapped.org** ```bash $ docker tag docker.io/registry:latest registry.airgapped.org/registry:latest $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE registry.airgapped.org/registry latest 177391bcf802 2 weeks ago 33.26 MB docker.io/registry latest 177391bcf802 2 weeks ago 33.26 MB ``` 3.3.2. Push newly taged image to **registry.airgapped.org** ```bash docker push registry.airgapped.org/registry ``` 3.3.3. Remove cached image on local docker, newly tag **registry.airgapped.org/registry:latest** ```bash docker rmi registry.airgapped.org/registry:latest $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE docker.io/registry latest 177391bcf802 2 weeks ago 33.26 MB ``` 3.3.4. Pull image **registry** from **registry.airgapped.org** ```bash $ docker pull registry.airgapped.org/registry Using default tag: latest Trying to pull repository registry.airgapped.org/registry ... latest: Pulling from registry.airgapped.org/registry Digest: sha256:e82c444f6275eaca07889d471943668ac17fd03ea8d863289a54c199ed216332 $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE docker.io/registry latest 177391bcf802 2 weeks ago 33.26 MB registry.airgapped.org/registry latest 177391bcf802 2 weeks ago 33.26 MB ``` **Now, you have a working and verified private registry!** For securer docker access, you shall follow the instruction list on [Restricting access](https://docs.docker.com/registry/deploying/#restricting-access).
json metadata{"tags":["docker","depolyment","air-gapped"],"links":["https://docs.docker.com/engine/installation/linux/linux-postinstall/#manage-docker-as-a-non-root-user","https://kubernetes.io/docs/concepts/cluster-administration/certificates/#openssl","https://docs.docker.com/registry/deploying/#restricting-access"],"app":"steemit/0.1","format":"markdown"}
parent author
parent permlinkdocker
permlinkdeploy-a-private-docker-registry-in-air-gapped-environment
titleDeploy a private docker registry in air-gapped environment
Transaction InfoBlock #19492368/Trx d287a01a9da02ca9b95d175e8ce1b1ae424c1978
View Raw JSON Data
{
  "block": 19492368,
  "op": [
    "comment",
    {
      "author": "highland0971",
      "body": "# Prepare required Docker environments\n\n## 1. Pre-defined environment\n- Internet availabe host: **localhost**\n- Air-gaped host: **master01.airgapped.org**\n\n## 2. Pre-request\n- You have to use the same linux distribution with your air-gaped host, which is  **master01.airgapped.org**, in this artical we adopt **CentOS 7** as host system.\n- You'd have to obtain a **PURE** linux environment which **HAVE** Internet access without any post-installed packages as was installed in **master01.airgapped.org**.  Otherwise you may encounter failed dependencies for docker in the **master01.airgapped.org**.\n- We assume that you have a **PURE** Internet access availabe host named **localhost**.\n### 2.1 Download and install the required packages for Docker\n-  On the **localhost** , run following commands:\n  ```bash\n  mkdir ~/rpms\n  sudo yum update && yum install -y --downloadonly  --downloaddir=${HOME}/rpms docker\n  ssh [email protected] mkdir packages\n  #You can verify the outcome by type\n  ssh [email protected] \"pwd && ls\"\n  scp ${HOME}/rpms/* [email protected]:~/packages\n  ```\n-  On the **master01.airgapped.org** , run following commands to install Docker environment\n  ```bash\n  cd ~/packages\n  sudo rpm -ivh *\n  ```\n  ​If you encoutner missing depandency error like\n  > warning: audit-libs-python-2.7.6-3.el7.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID f4a80eb5: NOKEY\n  > error: Failed dependencies:\n  >    libyajl.so.2()(64bit) is needed by oci-systemd-hook-1:0.1.14-1.git1ba44c6.el7.x86_64\n  >    libyajl.so.2()(64bit) is needed by oci-umount-2:2.3.0-1.git51e7c50.el7.x86_64\n  >    libcgroup is needed by policycoreutils-python-2.5-17.1.el7.x86_64\n  ​On the **localhost** you can run\n  ```bash\n  sudo yum update && yum reinstall -y --downloadonly  --downloaddir=${HOME}/rpms missing-packages(like yajl libcgroup)\n  ```\n  ​And redo the scp command for the missing rpms.\n### 2.2 Setup Docker environment and Shoot!\n-  On the **master01.airgapped.org** , following the procedure defined in [Manage Docker as a non-root user](https://docs.docker.com/engine/installation/linux/linux-postinstall/#manage-docker-as-a-non-root-user) to add docker user group.\n  ```bash\n  sudo groupadd docker\n  sudo usermod -aG docker $USER\n  ```\n  You have to re-login to make current $USER group affect.\n-  On the **master01.airgapped.org** , disable SeLinux in order to allow docker container access the host path file.\n  ```bash\n  sudo setenforce 0\n  sudo sed 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config > /etc/selinux/config\n  ```\n  Verify the whether selinux is disabled:\n  ```bash\n  $ getenforce\n  Permissive\n  ```\n-  On the **master01.airgapped.org** , enable and start the docker deamon.\n  ```bash\n  sudo systemctl enable docker && sudo systemctl start docker\n  ```\n-  On the **master01.airgapped.org** , check whether the docker's cgroupdriver is set to systemd `--exec-opt native.cgroupdriver=systemd`\n  ```bash\n  cat  /usr/lib/systemd/system/docker.service\n  ```\n  If not:\n  - Edit the `/usr/lib/systemd/system/docker.service` file and add `--exec-opt native.cgroupdriver=systemd` to `/usr/lib/systemd/system/docker.service`'s ExecStart strings.\n  - Restart the docker deamon service with\n    ```bash\n    sudo systemctl reload-deamon docker && sudo systemctl restart docker\n    ```\n-  On the **master01.airgapped.org** , check whether docker start successfully.\n    ```bash\n    systemctl status docker\n    ```\n#  Setup private registry service\n## 3. Pre-request\n- You have pulled the newset docker registry image from docker.io and export it to the tar archive.\n   ```bash\n     docker pull registry \\\n     && docker save docker.io/registry:latest -o docker-io.registry.tar\n   ```\n- You have to import the registry image to the **master01.airgapped.org** docker image repo cache\n  On **master01.airgapped.org**\n   ```bash\n    docker load -i docker-io.registry.tar\n   ```\n### 3.1 Create HTTPS certificates required for docker private registry\nThis procedure follows the example on [Openssl certificate creation on k8s official document](https://kubernetes.io/docs/concepts/cluster-administration/certificates/#openssl)\nOn **master01.airgapped.org** ：\n-  Prepare cert storage directory\n     ```bash\n     sudo mkdir /usr/lib/certs -p\n     pushd /usr/lib/certs\n     ```\n-  Generate CA key file\n     ```bash\n     sudo openssl genrsa -out ca.key 2048\n     ```\n-  Generate CA certificate file\n    ```bash\n    sudo openssl req -x509 -new -nodes -key ca.key -subj \"/CN=${MASTER_IP}\" -days 10000 -out ca.crt\n    ```\n  You can replace `${MASTER_IP}` with your actual or persuade CA server ip address  like\n  ```bash\n      sudo openssl req -x509 -new -nodes -key ca.key -subj \"/CN=8.8.8.8\" -days 10000 -out ca.crt\n  ```\n-  Generate docker docker registry server key file\n  ```bash\n  sudo openssl genrsa -out private-registry-server.key 2048\n  ```\n-  Create a config file for generating a Certificate Signing Request (CSR) for  docker registry server\n  ```bash\n  cat > /tmp/private-registry-server.conf <<EOF\n  [ req ]\n  default_bits = 2048\n  prompt = no\n  default_md = sha256\n  req_extensions = req_ext\n  distinguished_name = dn\n\n  [ dn ]\n  C = cn #replace with actual info\n  ST = Tianjin #replace with actual info\n  L = Tianjin #replace with actual info\n  O = Air Gaped Company #replace with actual info\n  OU = Air Gaped team #replace with actual info\n  CN = master01.airgapped.org  #replace with actual registry server name\n\n  [ req_ext ]\n  subjectAltName = @alt_names\n\n  [ alt_names ]\n  DNS.1 = master01.airgapped.org #replace with alternative dns resolve name\n  DNS.2 = registry.airgapped.org #replace with alternative dns resolve name\n  IP.1 = 192.168.122.163 #replace with actuel ip for the registry server\n\n  [ v3_ext ]\n  authorityKeyIdentifier=keyid,issuer:always\n  basicConstraints=CA:FALSE\n  keyUsage=keyEncipherment,dataEncipherment\n  extendedKeyUsage=serverAuth,clientAuth\n  subjectAltName=@alt_names\n  EOF\n\n  sudo scp /tmp/private-registry-server.conf ./\n  ```\n-  Generate the certificate signing request based on the config file for docker registry server\n  ```bash\n  sudo openssl req -new -key  private-registry-server.key \\\n  -out private-registry-server.csr \\\n  -config private-registry-server.conf\n  ```\n-  Sign the registry server certificate with CA cert and docker registry server CSR file\n  ```bash\n  sudo openssl x509 -req -in private-registry-server.csr -CA ca.crt -CAkey ca.key \\\n  -CAcreateserial -out private-registry-server.crt -days 10000 \\\n  -extensions v3_ext -extfile private-registry-server.conf\n  ```\n-  Verify the final cert for registry server\n  ```bash\n  openssl x509  -noout -text -in private-registry-server.crt\n  ```\n### 3.2 Start private registry\n-  Copy the CA certificate to docker's certs directory to avoid insecur repository error, the directory which certs resides in must have the same hostname with the private registry domain name.\n  ```bash\n  sudo mkdir /etc/docker/certs.d/registry.airgapped.org -p\n  sudo cp /usr/lib/certs/ca.crt /etc/docker/certs.d/registry.airgapped.org\n  ```\n-  Run registry container with preset certs and port config\n  ```bash\n  sudo mkdir /mnt/docker_images\n  docker run -d --restart=always -v /mnt/docker_images:/var/lib/registry \\\n     -v /usr/lib/certs:/cert \\\n     -e REGISTRY_HTTP_TLS_CERTIFICATE=/cert/private-registry-server.crt \\\n     -e REGISTRY_HTTP_TLS_KEY=/cert/private-registry-server.key \\\n     -e REGISTRY_HTTP_ADDR=0.0.0.0:443 \\\n     -p 443:443 --name local-registry registry\n  ```\n-  Verify private registry successfully started and listen on assigned port\n    ```bash\n    docker logs local-registry\n    ```\n    You should see the following output like,with **listening on [::]:443**\n    > time=\"2017-12-18T12:57:53Z\" level=warning msg=\"No HTTP secret provided - generated random secret. This may cause problems with uploads if multiple registries are behind a load-balancer. To provide a shared secret, fill in http.secret in the configuration file or set the REGISTRY_HTTP_SECRET environment variable.\" go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2\n    > time=\"2017-12-18T12:57:53Z\" level=info msg=\"redis not configured\" go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2\n    > time=\"2017-12-18T12:57:53Z\" level=info msg=\"Starting upload purge in 37m0s\" go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2\n    > time=\"2017-12-18T12:57:54Z\" level=info msg=\"using inmemory blob descriptor cache\" go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2\n    > time=\"2017-12-18T12:57:54Z\" level=info **msg=\"listening on [::]:443, tls\"** go.version=go1.7.6 instance.id=df382424-0e5d-49ad-b758-2b7b3e92d32f version=v2.6.2\n-  Append **registry.airgapped.org** domain to /etc/hosts, as dedicate domain for registry service\n  ```bash\n  sudo sh -c 'echo \"192.168.122.163 registry.airgapped.org\" >> /etc/hosts'\n  ```\n  If every thing is ok, you shall see the result by verifing the registry v2 API\n  ```bash\n  curl --cacert /usr/lib/certs/ca.crt https://registry.airgapped.org/v2/_catalog\n  {\"repositories\":[]}\n  ```\n## 3.3 Push your first image to private registry and verify\nOn **master01.airgapped.org**, At this point, you have:\n- one available private registry which is host on **master01.airgapped.org** and with a dedicate domain name **registry.airgapped.org** with TLS enabled, listening on port 443.\n- one local image named **docker.io/registry:latest** cached on **master01.airgapped.org** , as:\n  ```bash\n  $ docker images\n  REPOSITORY                       TAG                 IMAGE ID            CREATED             SIZE\n  docker.io/registry               latest              177391bcf802        2 weeks ago         33.26 MB\n  ```\n Now you can verify the service by the following scripts:\n  3.3.1. **Tag cached image with new registry path** which leads to **registry.airgapped.org**\n    ```bash\n    $ docker tag docker.io/registry:latest registry.airgapped.org/registry:latest\n    $ docker images\n    REPOSITORY                       TAG                 IMAGE ID            CREATED             SIZE\n    registry.airgapped.org/registry   latest              177391bcf802        2 weeks ago         33.26 MB\n    docker.io/registry               latest              177391bcf802        2 weeks ago         33.26 MB\n    ```\n 3.3.2. Push newly taged image to **registry.airgapped.org**\n    ```bash\n    docker push registry.airgapped.org/registry\n    ```\n 3.3.3. Remove cached image on local docker, newly tag **registry.airgapped.org/registry:latest**\n    ```bash\n    docker rmi registry.airgapped.org/registry:latest\n    $ docker images\n    REPOSITORY           TAG                 IMAGE ID            CREATED             SIZE\n    docker.io/registry   latest              177391bcf802        2 weeks ago         33.26 MB\n    ```\n 3.3.4. Pull image **registry** from **registry.airgapped.org**\n    ```bash\n    $ docker pull registry.airgapped.org/registry\n    Using default tag: latest\n    Trying to pull repository registry.airgapped.org/registry ...\n    latest: Pulling from registry.airgapped.org/registry\n    Digest: sha256:e82c444f6275eaca07889d471943668ac17fd03ea8d863289a54c199ed216332\n\n    $ docker images\n    REPOSITORY                       TAG                 IMAGE ID            CREATED             SIZE\n    docker.io/registry               latest              177391bcf802        2 weeks ago         33.26 MB\n    registry.airgapped.org/registry   latest              177391bcf802        2 weeks ago         33.26 MB\n    ```\n**Now, you have a working and verified private registry!**\n\nFor securer docker access, you shall follow the instruction list on [Restricting access](https://docs.docker.com/registry/deploying/#restricting-access).",
      "json_metadata": "{\"tags\":[\"docker\",\"depolyment\",\"air-gapped\"],\"links\":[\"https://docs.docker.com/engine/installation/linux/linux-postinstall/#manage-docker-as-a-non-root-user\",\"https://kubernetes.io/docs/concepts/cluster-administration/certificates/#openssl\",\"https://docs.docker.com/registry/deploying/#restricting-access\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}",
      "parent_author": "",
      "parent_permlink": "docker",
      "permlink": "deploy-a-private-docker-registry-in-air-gapped-environment",
      "title": "Deploy a private docker registry in air-gapped environment"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T15:40:54",
  "trx_id": "d287a01a9da02ca9b95d175e8ce1b1ae424c1978",
  "trx_in_block": 25,
  "virtual_op": 0
}
highland0971published a new post: asus-merlin-raspberrypi
2018/02/01 15:28:06
19,492,113|ec454b4
authorhighland0971
body在上一篇文章[《基于无预置IP list的GFW IP解锁方法》](https://steemit.com/cn/@highland0971/ip-list-gfw-ip)中讲述了如何通过Iptables自动检测受干扰的IP地址，在这里将进一步将相关代码整合到实际路由器中。前期个人买了一个Asus Merlin AC-87U，本想将Shadowsocks编译进路由器中，直接实现路由器内的透明代理转发，但尝试半天后，始终未能编译。于是考虑用RaspberryPi+Shadowsocks外挂AC-87U下实现路由转发。 具体实现步骤如下： ## 1. Asus Merlin AC-87U 干扰IP监测+自动策略路由配置 [《基于无预置IP list的GFW IP解锁方法》](https://steemit.com/cn/@highland0971/ip-list-gfw-ip)文中已经提到，实现无IP List的受干扰IP监测和路由转发主要分为两个步骤： - 在Iptables中配置GFW干扰IP监测策略 - 将检测到的干扰IP纳入策略路由中 由于华硕Merlin固件仅支持在特定JFFS分区下进行用户数据读写，同时仅允许在系统启动时在`/jffs/scripts`和`/jffs/configs`加载指定脚本和配置数据。因此上述两个关键步骤，就需要在`/jffs/scripts`目录下的指定脚本中进行配置。具体如下： ### 1.1 在`/jffs/scripts/firewall-start`脚本中配置iptables过滤策略 为了确保受干扰IP监测策略不被其他系统内置策略覆盖或影响，需要在路由器启动后并配置iptables默认策略后，将我们所需要的监测策略加入到Iptables的FORWARD链中,生效后通过`iptables-save`查看输出，应该可以看到在FORWARD链中已经增加了我们所需的4条指令。 ```iptables # Generated by iptables-save v1.4.14 on Thu Sep 21 21:38:26 2017 *filter :INPUT ACCEPT [7225:1346582] :FORWARD ACCEPT [0:0] :OUTPUT ACCEPT [7504:1290180] ..... ################### #一定要确保以下4条FORWARD规则是在FORWARD链的最前端，否则会被后续的规则直接bypass -A FORWARD -i eth0 -p tcp -m tcp --tcp-flags RST RST -j LOG --log-prefix "GFW_DECT_RST " -A FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN -j LOG --log-prefix "GFW_DEBUG " -A FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN -m recent --set --name syn_watch_list --rdest -A FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN -m recent --rcheck --seconds 15 --hitcount 3 --name syn_watch_list --rdest -j LOG --log-prefix "GFW_DECT_SYN " #################### -A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT -A FORWARD ! -i br0 -o eth0 -j DROP -A FORWARD -i eth0 -m state --state INVALID -j DROP -A FORWARD -i br0 -o br0 -j ACCEPT -A FORWARD -j NSFW -A FORWARD -m conntrack --ctstate DNAT -j ACCEPT -A FORWARD -i br0 -j ACCEPT ..... COMMIT # Completed on Thu Sep 21 21:38:26 2017 ``` ### 1.2 在`/jffs/scripts/service-start`脚本中配置策略路由 由于在本方案中AC-87U路由器并未内置[Shadowsocks](http://shadowsocks.org),无法实现路由器内的透明代理，因此我们需要将经iptables监测到的疑似受干扰目标ip的下一跳路由调整到装了[Shadowsocks](http://shadowsocks.org)的路由器中，也就是本文的RaspberryPi中。 为了实现这个目标，需要借助watch命令每秒钟刷新一次内核日志，过滤出需要做策略路由的ip地址，相关的watch指令，写在所有服务启动后的`/jffs/service-start`脚本中: ```bash #export GFW_LOCAL_GATEWAY=[Replace with your free Internat access gateway ip] export GFW_LOCAL_GATEWAY=192.168.9.137 export GFW_REMOTE_VPN=[Replace with your remote ss-server ip] #注意需要避免将到VPN路由地址[GFW_REMOTE_VPN]指向RaspberryPi路由器的地址 watch -n 1 dmesg -c|grep GFW_DECT|grep -v $GFW_REMOTE_VPN|\ awk '/GFW_DECT_SYN/{print $5};/GFW_DECT_RST/{print $4}'|\ awk -F= '{if($1=="SRC"){cmd="route -n|grep "$2" >/dev/null 2>&1 || \ route add "$2" gateway "ENVIRON["GFW_LOCAL_GATEWAY"]" >/dev/null 2>&1"} \ else{cmd="route -n|grep "$2" || ping -c 1 -W 1 "$2" >/dev/null 2>&1 || \ route add "$2" gateway "ENVIRON["GFW_LOCAL_GATEWAY"]" >/dev/null 2>&1"}; \ print(cmd);system(cmd)}' ``` 配置完重启路由器后，在电脑上浏览几个被屏蔽的网站，然后在AC-87U中通过route命令进行观察，便可以看到被干扰的IP地址，已正确的加入了策略路由中，转发给RaspberryPi路由器(记录略多哈 :-P )： ``` Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface 172.217.5.65 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 50.19.85.98 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 172.217.5.68 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 17.249.28.69 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 39.155.151.5 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 104.19.192.102 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 104.18.55.167 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 74.125.170.72 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 50.19.240.106 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 172.217.5.74 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 74.125.170.74 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 117.121.27.11 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 23.210.109.43 192.168.9.137 255.255.255.255 UGH 0 0 0 br0 ...... 169.254.39.0 * 255.255.255.0 U 0 0 0 br0 192.168.1.0 * 255.255.255.0 U 0 0 0 eth0 192.168.9.0 * 255.255.255.0 U 0 0 0 br0 127.0.0.0 * 255.0.0.0 U 0 0 0 lo default 192.168.1.1 0.0.0.0 UG 0 0 0 eth0 ``` ## 2. RaspberryPi+Shadowsocks透明代理 在已编译好Shadowsocks的RaspberryPi中的/etc/rc.local中配置透明转发规则： ```bash #!/bin/sh -e # # rc.local # Print the IP address _IP=$(hostname -I) || true if [ "$_IP" ]; then printf "My IP address is %s\n" "$_IP" fi export GFW_LOCAL_GATEWAY=[Replace with your free Internat access gateway ip] export GFW_REMOTE_VPN=[Replace with your remote ss-server ip] sudo ss-nat -s $GFW_REMOTE_VPN -l your_ss_redir_local_port -b $GFW_REMOTE_VPN -u sudo nohup ss-redir -s $GFW_REMOTE_VPN -p your_ss_server_passwd_listening_port \ -k your_ss_server_passwd -m your_ss_server_crypto_method \ -l your_ss_redir_local_port -b 0.0.0.0 -u exit 0 ``` 至此，AC-87U已经可以与RaspberryPi配合自动监测受干扰的IP地址，并进行策略路由，并进一步进行透明代理转发。 ## 3. Asus Merlin AC-87U DNS策略查询 当然，为了实现自由上网，除对IP地址做策略路由和透明代理外，还需进一步保护DNS不受污染，在这里感谢[Felix Yan](mailto:[email protected])同学提供的[dnsmasq-china-list](https://github.com/felixonmars/dnsmasq-china-list)工具，可以让路由器对不同的域名进行策略查询。 - 将dnsmasq-china-list压缩包从github上下载后，解压到AC-87U本地的/jffs/dnsmasq目录下 - 在/jffs/scripts/init-start脚本中配置主机启动后策略DNS解析文件自动拷贝 ```bash #!/bin/sh mkdir /etc/dnsmasq.d cp /jffs/dnsmasq/*.conf /etc/dnsmasq.d/ ``` - 配置在WAN口启动后(`/jffs/scripts/wan-start`)，将`/etc/dnsmasq.d/`中的指定域名DNS解析地址调整为WAN口上游分配的DNS IP地址 ```bash #!/bin/sh /jffs/dnsmasq/dnsmasq-update-china-list `cat /tmp/resolv.conf |awk '{print $2}'` ``` - 在/jffs/configs/目录中创建`dnsmasq.conf.add`文件，对默认dnsmasq启用`conf-dir`，并将默认DNS查询地址修改为不受干扰的DNS解析服务器 ```bash admin@RT-AC87U:/jffs/configs# cat dnsmasq.conf.add no-resolv server=YOUR_TRUSTY_DNS_IP#PORT conf-dir=/etc/dnsmasq.d ``` 至此，大功告成，Asus Merlin AC-87U+RaspberryPi+Shadowsocks的组合实现了自动识别被干扰IP+策略路由+透明代理+策略DNS解析，再次驰骋在广阔无垠的Internet上。
json metadata{"tags":["gfw","shadowsocks","cn","anti-gfw"],"links":["https://steemit.com/cn/@highland0971/ip-list-gfw-ip","http://shadowsocks.org","mailto:[email protected]","https://github.com/felixonmars/dnsmasq-china-list"],"app":"steemit/0.1","format":"markdown"}
parent author
parent permlinkgfw
permlinkasus-merlin-raspberrypi
title基于Asus Merlin+RaspberryPI打造无感知科学上网路由
Transaction InfoBlock #19492113/Trx ec454b4fdd2ba293622aec154920b5fb277e22ba
View Raw JSON Data
{
  "block": 19492113,
  "op": [
    "comment",
    {
      "author": "highland0971",
      "body": "在上一篇文章[《基于无预置IP list的GFW IP解锁方法》](https://steemit.com/cn/@highland0971/ip-list-gfw-ip)中讲述了如何通过Iptables自动检测受干扰的IP地址，在这里将进一步将相关代码整合到实际路由器中。前期个人买了一个Asus Merlin AC-87U，本想将Shadowsocks编译进路由器中，直接实现路由器内的透明代理转发，但尝试半天后，始终未能编译。于是考虑用RaspberryPi+Shadowsocks外挂AC-87U下实现路由转发。\n\n具体实现步骤如下：\n\n## 1. Asus Merlin AC-87U 干扰IP监测+自动策略路由配置\n\n[《基于无预置IP list的GFW IP解锁方法》](https://steemit.com/cn/@highland0971/ip-list-gfw-ip)文中已经提到，实现无IP List的受干扰IP监测和路由转发主要分为两个步骤：\n\n- 在Iptables中配置GFW干扰IP监测策略\n- 将检测到的干扰IP纳入策略路由中\n\n由于华硕Merlin固件仅支持在特定JFFS分区下进行用户数据读写，同时仅允许在系统启动时在`/jffs/scripts`和`/jffs/configs`加载指定脚本和配置数据。因此上述两个关键步骤，就需要在`/jffs/scripts`目录下的指定脚本中进行配置。具体如下：\n\n### 1.1  在`/jffs/scripts/firewall-start`脚本中配置iptables过滤策略\n\n为了确保受干扰IP监测策略不被其他系统内置策略覆盖或影响，需要在路由器启动后并配置iptables默认策略后，将我们所需要的监测策略加入到Iptables的FORWARD链中,生效后通过`iptables-save`查看输出，应该可以看到在FORWARD链中已经增加了我们所需的4条指令。\n\n```iptables\n# Generated by iptables-save v1.4.14 on Thu Sep 21 21:38:26 2017\n*filter\n:INPUT ACCEPT [7225:1346582]\n:FORWARD ACCEPT [0:0]\n:OUTPUT ACCEPT [7504:1290180]\n.....\n\n###################\n#一定要确保以下4条FORWARD规则是在FORWARD链的最前端，否则会被后续的规则直接bypass\n\n-A FORWARD -i eth0 -p tcp -m tcp --tcp-flags RST RST -j LOG --log-prefix \"GFW_DECT_RST \"\n-A FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN -j LOG --log-prefix \"GFW_DEBUG \"\n-A FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN -m recent --set --name syn_watch_list --rdest\n-A FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN -m recent --rcheck --seconds 15 --hitcount 3 --name syn_watch_list --rdest -j LOG --log-prefix \"GFW_DECT_SYN \"\n\n####################\n\n-A FORWARD -m state --state RELATED,ESTABLISHED -j ACCEPT\n-A FORWARD ! -i br0 -o eth0 -j DROP\n-A FORWARD -i eth0 -m state --state INVALID -j DROP\n-A FORWARD -i br0 -o br0 -j ACCEPT\n-A FORWARD -j NSFW\n-A FORWARD -m conntrack --ctstate DNAT -j ACCEPT\n-A FORWARD -i br0 -j ACCEPT\n.....\nCOMMIT\n# Completed on Thu Sep 21 21:38:26 2017\n\n```\n\n### 1.2 在`/jffs/scripts/service-start`脚本中配置策略路由\n\n由于在本方案中AC-87U路由器并未内置[Shadowsocks](http://shadowsocks.org),无法实现路由器内的透明代理，因此我们需要将经iptables监测到的疑似受干扰目标ip的下一跳路由调整到装了[Shadowsocks](http://shadowsocks.org)的路由器中，也就是本文的RaspberryPi中。\n\n为了实现这个目标，需要借助watch命令每秒钟刷新一次内核日志，过滤出需要做策略路由的ip地址，相关的watch指令，写在所有服务启动后的`/jffs/service-start`脚本中:\n```bash\n#export GFW_LOCAL_GATEWAY=[Replace with your free Internat access gateway ip]\nexport GFW_LOCAL_GATEWAY=192.168.9.137\nexport GFW_REMOTE_VPN=[Replace with your remote ss-server ip]\n\n#注意需要避免将到VPN路由地址[GFW_REMOTE_VPN]指向RaspberryPi路由器的地址\nwatch -n 1 dmesg -c|grep GFW_DECT|grep -v $GFW_REMOTE_VPN|\\\n        awk '/GFW_DECT_SYN/{print $5};/GFW_DECT_RST/{print $4}'|\\\n        awk -F= '{if($1==\"SRC\"){cmd=\"route -n|grep \"$2\" >/dev/null 2>&1 || \\\n                route add \"$2\" gateway \"ENVIRON[\"GFW_LOCAL_GATEWAY\"]\" >/dev/null 2>&1\"} \\\n                else{cmd=\"route -n|grep \"$2\" || ping -c 1 -W 1 \"$2\"  >/dev/null 2>&1 || \\\n                route add \"$2\" gateway \"ENVIRON[\"GFW_LOCAL_GATEWAY\"]\" >/dev/null 2>&1\"}; \\\n                print(cmd);system(cmd)}'\n```\n配置完重启路由器后，在电脑上浏览几个被屏蔽的网站，然后在AC-87U中通过route命令进行观察，便可以看到被干扰的IP地址，已正确的加入了策略路由中，转发给RaspberryPi路由器(记录略多哈 :-P )：\n```\nKernel IP routing table\nDestination     Gateway         Genmask         Flags Metric Ref    Use Iface\n172.217.5.65    192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n50.19.85.98     192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n172.217.5.68    192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n17.249.28.69    192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n39.155.151.5    192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n104.19.192.102  192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n104.18.55.167   192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n74.125.170.72   192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n50.19.240.106   192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n172.217.5.74    192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n74.125.170.74   192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n117.121.27.11   192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n23.210.109.43   192.168.9.137   255.255.255.255 UGH   0      0        0 br0\n......\n169.254.39.0    *               255.255.255.0   U     0      0        0 br0\n192.168.1.0     *               255.255.255.0   U     0      0        0 eth0\n192.168.9.0     *               255.255.255.0   U     0      0        0 br0\n127.0.0.0       *               255.0.0.0       U     0      0        0 lo\ndefault         192.168.1.1     0.0.0.0         UG    0      0        0 eth0\n\n```\n\n## 2. RaspberryPi+Shadowsocks透明代理\n\n在已编译好Shadowsocks的RaspberryPi中的/etc/rc.local中配置透明转发规则：\n\n```bash\n\n#!/bin/sh -e\n#\n# rc.local\n\n# Print the IP address\n_IP=$(hostname -I) || true\nif [ \"$_IP\" ]; then\n  printf \"My IP address is %s\\n\" \"$_IP\"\nfi\n\nexport GFW_LOCAL_GATEWAY=[Replace with your free Internat access gateway ip]\nexport GFW_REMOTE_VPN=[Replace with your remote ss-server ip]\nsudo ss-nat -s $GFW_REMOTE_VPN -l your_ss_redir_local_port -b $GFW_REMOTE_VPN -u\nsudo nohup ss-redir -s $GFW_REMOTE_VPN -p your_ss_server_passwd_listening_port \\\n                    -k your_ss_server_passwd -m your_ss_server_crypto_method \\\n                    -l your_ss_redir_local_port -b 0.0.0.0 -u\n\nexit 0\n\n```\n至此，AC-87U已经可以与RaspberryPi配合自动监测受干扰的IP地址，并进行策略路由，并进一步进行透明代理转发。\n\n## 3. Asus Merlin AC-87U DNS策略查询\n\n当然，为了实现自由上网，除对IP地址做策略路由和透明代理外，还需进一步保护DNS不受污染，在这里感谢[Felix Yan](mailto:[email protected])同学提供的[dnsmasq-china-list](https://github.com/felixonmars/dnsmasq-china-list)工具，可以让路由器对不同的域名进行策略查询。\n\n- 将dnsmasq-china-list压缩包从github上下载后，解压到AC-87U本地的/jffs/dnsmasq目录下\n\n- 在/jffs/scripts/init-start脚本中配置主机启动后策略DNS解析文件自动拷贝\n\n```bash\n#!/bin/sh\n\nmkdir /etc/dnsmasq.d\ncp /jffs/dnsmasq/*.conf /etc/dnsmasq.d/\n```\n\n- 配置在WAN口启动后(`/jffs/scripts/wan-start`)，将`/etc/dnsmasq.d/`中的指定域名DNS解析地址调整为WAN口上游分配的DNS IP地址\n\n```bash\n#!/bin/sh\n\n/jffs/dnsmasq/dnsmasq-update-china-list `cat /tmp/resolv.conf |awk '{print $2}'`\n```\n\n- 在/jffs/configs/目录中创建`dnsmasq.conf.add`文件，对默认dnsmasq启用`conf-dir`，并将默认DNS查询地址修改为不受干扰的DNS解析服务器\n\n```bash\nadmin@RT-AC87U:/jffs/configs# cat dnsmasq.conf.add\nno-resolv\nserver=YOUR_TRUSTY_DNS_IP#PORT\nconf-dir=/etc/dnsmasq.d\n```\n\n至此，大功告成，Asus Merlin AC-87U+RaspberryPi+Shadowsocks的组合实现了自动识别被干扰IP+策略路由+透明代理+策略DNS解析，再次驰骋在广阔无垠的Internet上。",
      "json_metadata": "{\"tags\":[\"gfw\",\"shadowsocks\",\"cn\",\"anti-gfw\"],\"links\":[\"https://steemit.com/cn/@highland0971/ip-list-gfw-ip\",\"http://shadowsocks.org\",\"mailto:[email protected]\",\"https://github.com/felixonmars/dnsmasq-china-list\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}",
      "parent_author": "",
      "parent_permlink": "gfw",
      "permlink": "asus-merlin-raspberrypi",
      "title": "基于Asus Merlin+RaspberryPI打造无感知科学上网路由"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T15:28:06",
  "trx_id": "ec454b4fdd2ba293622aec154920b5fb277e22ba",
  "trx_in_block": 3,
  "virtual_op": 0
}
highland0971published a new post: ip-list-gfw-ip
2018/02/01 15:20:21
19,491,958|cbcd877
authorhighland0971
body在以往的基于路由、iptables解锁GFW IP封锁的方法中，均需要一份国内或国外的IP地址列表，然后进行策略路由。基于这种方案的解锁，需要人工去维护一张IP地址列表，其中基于[chnroutes](https://github.com/fivesheep/chnroutes)方案生成的国内IP地址列表就有8000多条记录,在加载的时候会比较耗时，同时给路由器带来一定负担。本文给出一种基于无IP列表的自动IP访问监测及路由更变策略。 ### 1. 修改iptables规则，实现对封锁IP或者干扰IP的自动检测标记 - 封锁 IP监测，对于被封锁的IP，客户端往往会在短时间内多次发起TCP SYN请求，通过捕获高频SYN请求，识别潜在的被封锁IP地址 ```bash iptables -I FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN \ -m recent --rcheck --seconds 15 --hitcount 3 --name syn_watch_list --rdest \ -j LOG --log-prefix "GFW_DECT_SYN " iptables -I FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN \ -m recent --set --name syn_watch_list --rdest iptables -I FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN \ -j LOG --log-prefix "GFW_DEBUG " ``` - 受干扰IP监测（敏感词过滤），对于被干扰的IP，通过捕获来自WAN口的RST请求，识别潜在被封锁的IP地址 ```bash iptables -I FORWARD -i eth0 -p tcp -m tcp --tcp-flags RST RST -j LOG --log-prefix "GFW_DECT_RST " ``` ### 2. 将监测到的被干扰IP加入策略路由（基于旁路路由）或iptables端口重定向(基于本地**ss-redir**实现) 这里给出的是基于旁路路由的实现策略 ```bash export GFW_LOCAL_GATEWAY=[Replace with your free Internat access gateway ip] export GFW_REMOTE_VPN=[Replace with your remote ss-server ip] watch -n 1 dmesg -c|grep GFW_DECT|grep -v $GFW_REMOTE_VPN|\ awk '/GFW_DECT_SYN/{print $5};/GFW_DECT_RST/{print $4}'|\ awk -F= '{if($1=="SRC"){cmd="route -n|grep "$2" >/dev/null 2>&1 || \ route add "$2" gateway "ENVIRON["GFW_LOCAL_GATEWAY"]" >/dev/null 2>&1"} \ else{cmd="route -n|grep "$2" || ping -c 1 -W 1 "$2" >/dev/null 2>&1 || \ route add "$2" gateway "ENVIRON["GFW_LOCAL_GATEWAY"]" >/dev/null 2>&1"}; \ print(cmd);system(cmd)}' ``` ### 3. 进行透明路由转发 在以上的配置中，是将识别出来的受干扰IP路由给了另外一个装着**shadowsocks**的旁路路由器，由其使用**ss-redir**进行透明转发配置如下： ```bash export GFW_LOCAL_GATEWAY=[Replace with your free Internat access gateway ip] export GFW_REMOTE_VPN=[Replace with your remote ss-server ip] sudo ss-nat -s $GFW_REMOTE_VPN -l [Your ss-redir local listening port] -b $GFW_REMOTE_VPN -u sudo nohup ss-redir -s $GFW_REMOTE_VPN -p [Your ss-server listening port] \ -k [Your ss-server password] -m [Your ss-server crypto method] \ -l [Your ss-redir local listening port] -b 0.0.0.0 -u ```
json metadata{"tags":["cn","gfw","iptables","anti-gfw"],"links":["https://github.com/fivesheep/chnroutes"],"app":"steemit/0.1","format":"markdown"}
parent author
parent permlinkcn
permlinkip-list-gfw-ip
title基于无预置IP list的GFW IP解锁方法
Transaction InfoBlock #19491958/Trx cbcd877aa73d177dc63bcf9be0089c04c9e9d9d3
View Raw JSON Data
{
  "block": 19491958,
  "op": [
    "comment",
    {
      "author": "highland0971",
      "body": "在以往的基于路由、iptables解锁GFW IP封锁的方法中，均需要一份国内或国外的IP地址列表，然后进行策略路由。基于这种方案的解锁，需要人工去维护一张IP地址列表，其中基于[chnroutes](https://github.com/fivesheep/chnroutes)方案生成的国内IP地址列表就有8000多条记录,在加载的时候会比较耗时，同时给路由器带来一定负担。本文给出一种基于无IP列表的自动IP访问监测及路由更变策略。\n\n### 1. 修改iptables规则，实现对封锁IP或者干扰IP的自动检测标记\n\n- 封锁 IP监测，对于被封锁的IP，客户端往往会在短时间内多次发起TCP SYN请求，通过捕获高频SYN请求，识别潜在的被封锁IP地址\n\n```bash\niptables -I FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN \\\n        -m recent --rcheck --seconds 15 --hitcount 3 --name syn_watch_list --rdest \\\n        -j LOG --log-prefix \"GFW_DECT_SYN \"\niptables -I FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN \\\n        -m recent --set --name syn_watch_list --rdest\niptables -I FORWARD -i br0 -o eth0 -p tcp -m tcp --tcp-flags SYN,ACK SYN \\\n        -j LOG --log-prefix \"GFW_DEBUG \"\n```\n- 受干扰IP监测（敏感词过滤），对于被干扰的IP，通过捕获来自WAN口的RST请求，识别潜在被封锁的IP地址\n\n```bash\niptables -I FORWARD -i eth0 -p tcp -m tcp --tcp-flags RST RST -j LOG --log-prefix \"GFW_DECT_RST \"\n```\n\n### 2. 将监测到的被干扰IP加入策略路由（基于旁路路由）或iptables端口重定向(基于本地**ss-redir**实现)\n这里给出的是基于旁路路由的实现策略\n```bash\nexport GFW_LOCAL_GATEWAY=[Replace with your free Internat access gateway ip]\nexport GFW_REMOTE_VPN=[Replace with your remote ss-server ip]\n\nwatch -n 1 dmesg -c|grep GFW_DECT|grep -v $GFW_REMOTE_VPN|\\\n        awk '/GFW_DECT_SYN/{print $5};/GFW_DECT_RST/{print $4}'|\\\n        awk -F= '{if($1==\"SRC\"){cmd=\"route -n|grep \"$2\" >/dev/null 2>&1 || \\\n                route add \"$2\" gateway \"ENVIRON[\"GFW_LOCAL_GATEWAY\"]\" >/dev/null 2>&1\"} \\\n                else{cmd=\"route -n|grep \"$2\" || ping -c 1 -W 1 \"$2\"  >/dev/null 2>&1 || \\\n                route add \"$2\" gateway \"ENVIRON[\"GFW_LOCAL_GATEWAY\"]\" >/dev/null 2>&1\"}; \\\n                print(cmd);system(cmd)}'\n```\n\n### 3. 进行透明路由转发\n\n在以上的配置中，是将识别出来的受干扰IP路由给了另外一个装着**shadowsocks**的旁路路由器，由其使用**ss-redir**进行透明转发配置如下：\n\n```bash\nexport GFW_LOCAL_GATEWAY=[Replace with your free Internat access gateway ip]\nexport GFW_REMOTE_VPN=[Replace with your remote ss-server ip]\nsudo ss-nat -s $GFW_REMOTE_VPN -l [Your ss-redir local listening port] -b $GFW_REMOTE_VPN -u\nsudo nohup ss-redir -s $GFW_REMOTE_VPN -p [Your ss-server listening port] \\\n                    -k [Your ss-server password] -m [Your ss-server crypto method] \\\n                    -l [Your ss-redir local listening port] -b 0.0.0.0 -u\n```",
      "json_metadata": "{\"tags\":[\"cn\",\"gfw\",\"iptables\",\"anti-gfw\"],\"links\":[\"https://github.com/fivesheep/chnroutes\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}",
      "parent_author": "",
      "parent_permlink": "cn",
      "permlink": "ip-list-gfw-ip",
      "title": "基于无预置IP list的GFW IP解锁方法"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T15:20:21",
  "trx_id": "cbcd877aa73d177dc63bcf9be0089c04c9e9d9d3",
  "trx_in_block": 160,
  "virtual_op": 0
}
eosgoupvoted (100.00%) @highland0971 / how-to-compile-eos-on-clean-centos-system
2018/02/01 11:55:39
19,487,871|0dae0e6
authorhighland0971
permlinkhow-to-compile-eos-on-clean-centos-system
votereosgo
weight10000 (100.00%)
Transaction InfoBlock #19487871/Trx 0dae0e6ede058a88a49a02bfb77b3ea7de00cde7
View Raw JSON Data
{
  "block": 19487871,
  "op": [
    "vote",
    {
      "author": "highland0971",
      "permlink": "how-to-compile-eos-on-clean-centos-system",
      "voter": "eosgo",
      "weight": 10000
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T11:55:39",
  "trx_id": "0dae0e6ede058a88a49a02bfb77b3ea7de00cde7",
  "trx_in_block": 11,
  "virtual_op": 0
}
vetazabiraupvoted (100.00%) @highland0971 / how-to-compile-eos-on-clean-centos-system
2018/02/01 11:36:36
19,487,490|0e2698d
authorhighland0971
permlinkhow-to-compile-eos-on-clean-centos-system
votervetazabira
weight10000 (100.00%)
Transaction InfoBlock #19487490/Trx 0e2698dea8613dbfd85a7a4105644a025a9b5f27
View Raw JSON Data
{
  "block": 19487490,
  "op": [
    "vote",
    {
      "author": "highland0971",
      "permlink": "how-to-compile-eos-on-clean-centos-system",
      "voter": "vetazabira",
      "weight": 10000
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T11:36:36",
  "trx_id": "0e2698dea8613dbfd85a7a4105644a025a9b5f27",
  "trx_in_block": 32,
  "virtual_op": 0
}
eugene.chungupvoted (100.00%) @highland0971 / how-to-compile-eos-on-clean-centos-system
2018/02/01 07:39:48
19,482,767|9904673
authorhighland0971
permlinkhow-to-compile-eos-on-clean-centos-system
votereugene.chung
weight10000 (100.00%)
Transaction InfoBlock #19482767/Trx 9904673f05046373a0bdd04afbac3b6a463643d4
View Raw JSON Data
{
  "block": 19482767,
  "op": [
    "vote",
    {
      "author": "highland0971",
      "permlink": "how-to-compile-eos-on-clean-centos-system",
      "voter": "eugene.chung",
      "weight": 10000
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T07:39:48",
  "trx_id": "9904673f05046373a0bdd04afbac3b6a463643d4",
  "trx_in_block": 22,
  "virtual_op": 0
}
highland0971updated their account properties
2018/02/01 06:25:33
19,481,285|f701176
accounthighland0971
json metadata{"profile":{"name":"Highland","website":"https://highland0971.github.io/"}}
memo keySTM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp
Transaction InfoBlock #19481285/Trx f70117640f1cc898c7ec6f7dd5b462d470802cdb
View Raw JSON Data
{
  "block": 19481285,
  "op": [
    "account_update",
    {
      "account": "highland0971",
      "json_metadata": "{\"profile\":{\"name\":\"Highland\",\"website\":\"https://highland0971.github.io/\"}}",
      "memo_key": "STM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T06:25:33",
  "trx_id": "f70117640f1cc898c7ec6f7dd5b462d470802cdb",
  "trx_in_block": 46,
  "virtual_op": 0
}
highland0971followed @tokenika
2018/02/01 02:44:24
19,476,868|870de1f
idfollow
json["follow",{"follower":"highland0971","following":"tokenika","what":["blog"]}]
required auths[]
required posting auths["highland0971"]
Transaction InfoBlock #19476868/Trx 870de1ff2f3ca3914f5a0c7c9a50a1cfbe976016
View Raw JSON Data
{
  "block": 19476868,
  "op": [
    "custom_json",
    {
      "id": "follow",
      "json": "[\"follow\",{\"follower\":\"highland0971\",\"following\":\"tokenika\",\"what\":[\"blog\"]}]",
      "required_auths": [],
      "required_posting_auths": [
        "highland0971"
      ]
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T02:44:24",
  "trx_id": "870de1ff2f3ca3914f5a0c7c9a50a1cfbe976016",
  "trx_in_block": 10,
  "virtual_op": 0
}
highland0971followed @dantheman
2018/02/01 02:44:18
19,476,867|67427d1
idfollow
json["follow",{"follower":"highland0971","following":"dantheman","what":["blog"]}]
required auths[]
required posting auths["highland0971"]
Transaction InfoBlock #19476867/Trx 67427d12eaf7a0a794d8e8f141da53c03f20fe8a
View Raw JSON Data
{
  "block": 19476867,
  "op": [
    "custom_json",
    {
      "id": "follow",
      "json": "[\"follow\",{\"follower\":\"highland0971\",\"following\":\"dantheman\",\"what\":[\"blog\"]}]",
      "required_auths": [],
      "required_posting_auths": [
        "highland0971"
      ]
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T02:44:18",
  "trx_id": "67427d12eaf7a0a794d8e8f141da53c03f20fe8a",
  "trx_in_block": 63,
  "virtual_op": 0
}
highland0971followed @iang
2018/02/01 02:44:18
19,476,867|ad305ae
idfollow
json["follow",{"follower":"highland0971","following":"iang","what":["blog"]}]
required auths[]
required posting auths["highland0971"]
Transaction InfoBlock #19476867/Trx ad305aed5238d3ea733bd063993ab6aa5219ec34
View Raw JSON Data
{
  "block": 19476867,
  "op": [
    "custom_json",
    {
      "id": "follow",
      "json": "[\"follow\",{\"follower\":\"highland0971\",\"following\":\"iang\",\"what\":[\"blog\"]}]",
      "required_auths": [],
      "required_posting_auths": [
        "highland0971"
      ]
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T02:44:18",
  "trx_id": "ad305aed5238d3ea733bd063993ab6aa5219ec34",
  "trx_in_block": 9,
  "virtual_op": 0
}
highland0971followed @eosgo
2018/02/01 02:44:12
19,476,865|7112fad
idfollow
json["follow",{"follower":"highland0971","following":"eosgo","what":["blog"]}]
required auths[]
required posting auths["highland0971"]
Transaction InfoBlock #19476865/Trx 7112fad5e87f4842d9cd6db66994b1eb2c82f0f5
View Raw JSON Data
{
  "block": 19476865,
  "op": [
    "custom_json",
    {
      "id": "follow",
      "json": "[\"follow\",{\"follower\":\"highland0971\",\"following\":\"eosgo\",\"what\":[\"blog\"]}]",
      "required_auths": [],
      "required_posting_auths": [
        "highland0971"
      ]
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T02:44:12",
  "trx_id": "7112fad5e87f4842d9cd6db66994b1eb2c82f0f5",
  "trx_in_block": 37,
  "virtual_op": 0
}
highland0971followed @eosio
2018/02/01 02:44:00
19,476,861|5aa12b9
idfollow
json["follow",{"follower":"highland0971","following":"eosio","what":["blog"]}]
required auths[]
required posting auths["highland0971"]
Transaction InfoBlock #19476861/Trx 5aa12b9f0491b4403161e89a44d0d27fb3cfe2aa
View Raw JSON Data
{
  "block": 19476861,
  "op": [
    "custom_json",
    {
      "id": "follow",
      "json": "[\"follow\",{\"follower\":\"highland0971\",\"following\":\"eosio\",\"what\":[\"blog\"]}]",
      "required_auths": [],
      "required_posting_auths": [
        "highland0971"
      ]
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T02:44:00",
  "trx_id": "5aa12b9f0491b4403161e89a44d0d27fb3cfe2aa",
  "trx_in_block": 43,
  "virtual_op": 0
}
highland0971upvoted (100.00%) @highland0971 / how-to-compile-eos-on-clean-centos-system
2018/02/01 02:04:03
19,476,064|7b21add
authorhighland0971
permlinkhow-to-compile-eos-on-clean-centos-system
voterhighland0971
weight10000 (100.00%)
Transaction InfoBlock #19476064/Trx 7b21addbc23021976c75d09adc335f252818c6df
View Raw JSON Data
{
  "block": 19476064,
  "op": [
    "vote",
    {
      "author": "highland0971",
      "permlink": "how-to-compile-eos-on-clean-centos-system",
      "voter": "highland0971",
      "weight": 10000
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T02:04:03",
  "trx_id": "7b21addbc23021976c75d09adc335f252818c6df",
  "trx_in_block": 5,
  "virtual_op": 0
}
highland0971published a new post: how-to-compile-eos-on-clean-centos-system
2018/02/01 02:04:03
19,476,064|7b21add
authorhighland0971
bodyThere is a lot of posts about how to compile EOS on MacOS and Ubuntu, and the [EOS offical github site](https://github.com/EOSIO/eos) has given a complete compile procedure guidence & automatic scripts to help the developers. However the choice of operting system is subjected to the actual cluster we use in production environment, and the official did not give a clue how to compile on different systems. According to my own compile practice, there are a lot of bugs and traps that need to be avoid when you compile the EOS. To help the developers who are anxious to experience the EOS on their prefered operating system, I write the following article to demonstrate how I compile on CentOS 7 and how to avoid the traps that hidden under the [offical guide](https://github.com/EOSIO/eos/wiki/Local-Environment#2-building-eos). # 1. Hardware and System requirements Due to the requirements of building LLVM WebAssembly，the host must have 10GB free space and the RAM need to be greater than 16GB, multiple cpu cores is suggested. The operating system we use is CentOS 7.4 , you may choose other Redhat releases. # 2. Basic compile environment requirement. According to the description on EOS.IO github official site,the EOS project is composed on C++14 stander, the default compile come with CentOS 7 is GNU 4.8.5, thus we need to manual choose different repos to install higher verison of GNU compile toolchain. Besides, the CMake version in CentOS 7 official site is a bit old, we need to install the newest Cmake manually. ```bash export BUILD_TEMP=${HOME}/tmp mkdir ${BUILD_TEMP} ##GNU C++ 6.3.1 sudo yum install -y centos-release-scl sudo yum install -y devtoolset-6 ##CMake 3.10.2 export BUILD_TEMP=${HOME}/tmp mkdir ${BUILD_TEMP} cd ${BUILD_TEMP} mkdir ${HOME}/opt curl -L https://cmake.org/files/v3.10/cmake-3.10.2-Linux-x86_64.tar.gz -o cmake-3.10.2-Linux-x86_64.tar.gz tar -zxf cmake-3.10.2-Linux-x86_64.tar.gz -C ${HOME}/opt export PATH=$PATH:/opt/rh/devtoolset-6/root/bin/:${HOME}/opt/cmake-3.10.2-Linux-x86_64/bin/ ##Other compile tool chain sudo yum install -y git autoconf automake libtool doxygen ocaml gmp-devel python-devel bzip2-devel openssl-devel libicu-devel bzip2 wget ``` # 3. Compile the dependancy of EOS.IO According to the official site, compile EOS.IO requires following components: - Boost 1.64 - secp256k1-zkp - Binaryen - LLVM 4.0 with WebAssembly（On actual compile environment, you also have to compile X86 target for a clean CentOS installation） ## 3.1. Compile Boost ```bash export BOOST_ROOT=${HOME}/opt/boost_1_64_0 cd ${BUILD_TEMP} curl -L https://sourceforge.net/projects/boost/files/boost/1.64.0/boost_1_64_0.tar.bz2 > boost_1.64.0.tar.bz2 tar xvf boost_1.64.0.tar.bz2 cd boost_1_64_0/ ./bootstrap.sh "--prefix=$BOOST_ROOT" ./b2 install ``` ## 3.2. Compile secp256k1-zkp ```bash cd ${BUILD_TEMP} git clone https://github.com/cryptonomex/secp256k1-zkp.git cd secp256k1-zkp ./autogen.sh && ./configure make sudo make install ``` ## 3.3. Compile Binaryen ```bash cd ${BUILD_TEMP} git clone https://github.com/WebAssembly/binaryen cd binaryen git checkout tags/1.37.14 cmake . && make -j 4 sudo make install ``` ## 3.4. Compile LLVM WebAssembly+ Host Target - **TRAP 1** Due the EOS need LLVM to compile C++ to WebAssembly, the frontend of Clang and WebAssembly are both required, the official compile guidance only compile the WebAssembly target (I assume that is based on the clang is already installed on the ubuntu system,so the auther omitted it when compile LLVM for WebAssembly.), when we compile the LLVM wtih GNU compiler we need to both compile the WebAssembly and X86 target (which is achived by adding parameter `DLLVM_TARGETS_TO_BUILD='host'` to the cmake configuration), otherwise we would encounter missing `LLVMX86****.so` error when you compile EOS. - **TRAP 2** At current, the EOS only support LLVM of version 4.0,which is achived by clone branch 40 from [llvm mirror site](https://github.com/llvm-mirror/llvm.git), **DO NOT** directlly pull the latest source from github. - **TRAP 3** You have to enable RTTI when you compile, otherwise you may encounter error like ***typeof ** undefine*** when you compile EOS. You can avoid this error by add parameter `DLLVM_ENABLE_RTTI=ON` to the cmake configuration. ```bash cd ${BUILD_TEMP} mkdir ${BUILD_TEMP}/wasm-compiler/build -p cd ${BUILD_TEMP}/wasm-compiler git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git cd llvm/tools git clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/clang.git cd ${BUILD_TEMP}/wasm-compiler/build cmake -G "Unix Makefiles" -DLLVM_ENABLE_RTTI=ON -DLLVM_TARGETS_TO_BUILD='host' -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly -DCMAKE_BUILD_TYPE=Release ../llvm sudo make -j4 install export WASM_LLVM_CONFIG=/usr/local/bin/llvm-config ``` # 4. Compile EOS Finally, when you completed the all the dependancy compilation, you may proceed to compile the EOS source now. ```bash cd ${BUILD_TEMP} git clone https://github.com/eosio/eos --recursive mkdir -p ${BUILD_TEMP}/eos/build && cd ${BUILD_TEMP}/eos/build cmake -DBOOST_INCLUDEDIR=${BOOST_ROOT}/include/boost \ -DBOOST_LIBRARYDIR=${BOOST_ROOT}/lib .. make -j4 ``` # 5. Run your first EOS node! Please refer to [Creating and launching a single-node testnet](https://github.com/EOSIO/eos/wiki/Local-Environment#4-creating-and-launching-a-single-node-testnet).
json metadata{"tags":["eos"],"links":["https://github.com/EOSIO/eos","https://github.com/EOSIO/eos/wiki/Local-Environment#2-building-eos","https://github.com/llvm-mirror/llvm.git","https://github.com/EOSIO/eos/wiki/Local-Environment#4-creating-and-launching-a-single-node-testnet"],"app":"steemit/0.1","format":"markdown"}
parent author
parent permlinkeos
permlinkhow-to-compile-eos-on-clean-centos-system
titleHow to compile EOS on clean CentOS system
Transaction InfoBlock #19476064/Trx 7b21addbc23021976c75d09adc335f252818c6df
View Raw JSON Data
{
  "block": 19476064,
  "op": [
    "comment",
    {
      "author": "highland0971",
      "body": "There is a lot of posts about how to compile EOS on MacOS and Ubuntu, and the [EOS offical github site](https://github.com/EOSIO/eos) has given a complete compile procedure guidence & automatic scripts to help the developers. \nHowever the choice of operting system is subjected to the actual cluster we use in production environment, and the official did not give a clue how to compile on different systems. According to my own compile practice, there are a lot of bugs and traps that need to be avoid when you compile the EOS. To help the developers who are anxious to experience the EOS on their prefered operating system, I write the following article to demonstrate how I compile on CentOS 7 and how to avoid the traps that hidden under the [offical guide](https://github.com/EOSIO/eos/wiki/Local-Environment#2-building-eos).\n\n\n# 1. Hardware and System requirements\nDue to the requirements of building LLVM WebAssembly，the host must have 10GB free space and the RAM need to be greater than 16GB, multiple cpu cores is suggested.\n\nThe operating system we use is CentOS 7.4 , you may choose other Redhat releases.\n\n# 2. Basic compile environment requirement.\nAccording to the description on EOS.IO github official site,the EOS project is composed on C++14 stander, the default compile come with CentOS 7 is GNU 4.8.5, thus we need to manual choose different repos to install higher verison of GNU compile toolchain. Besides, the CMake version in  CentOS 7 official site is a bit old, we need to install the newest Cmake manually.\n\n```bash\nexport BUILD_TEMP=${HOME}/tmp\nmkdir ${BUILD_TEMP}\n\n##GNU C++ 6.3.1\nsudo yum install -y centos-release-scl\nsudo yum install -y devtoolset-6\n\n##CMake 3.10.2\n\nexport BUILD_TEMP=${HOME}/tmp\nmkdir ${BUILD_TEMP}\n\ncd  ${BUILD_TEMP}\nmkdir ${HOME}/opt\ncurl -L https://cmake.org/files/v3.10/cmake-3.10.2-Linux-x86_64.tar.gz -o cmake-3.10.2-Linux-x86_64.tar.gz\ntar -zxf cmake-3.10.2-Linux-x86_64.tar.gz -C ${HOME}/opt\n\nexport PATH=$PATH:/opt/rh/devtoolset-6/root/bin/:${HOME}/opt/cmake-3.10.2-Linux-x86_64/bin/\n\n##Other compile tool chain\nsudo yum install -y git autoconf automake libtool doxygen ocaml  gmp-devel python-devel bzip2-devel openssl-devel libicu-devel bzip2 wget\n\n```\n# 3. Compile the dependancy of EOS.IO\nAccording to the official site, compile EOS.IO requires following components:\n- Boost 1.64\n- secp256k1-zkp\n- Binaryen\n- LLVM 4.0 with WebAssembly（On actual compile environment, you also have to compile X86 target for a clean CentOS installation）\n## 3.1. Compile Boost\n```bash\nexport BOOST_ROOT=${HOME}/opt/boost_1_64_0\ncd  ${BUILD_TEMP}\ncurl -L https://sourceforge.net/projects/boost/files/boost/1.64.0/boost_1_64_0.tar.bz2 > boost_1.64.0.tar.bz2\ntar xvf boost_1.64.0.tar.bz2\ncd boost_1_64_0/\n./bootstrap.sh \"--prefix=$BOOST_ROOT\"\n./b2 install\n```\n## 3.2. Compile secp256k1-zkp\n```bash\ncd  ${BUILD_TEMP}\ngit clone https://github.com/cryptonomex/secp256k1-zkp.git\ncd secp256k1-zkp\n./autogen.sh && ./configure\nmake\nsudo make install\n```\n## 3.3. Compile Binaryen\n```bash\ncd  ${BUILD_TEMP}\ngit clone https://github.com/WebAssembly/binaryen\ncd binaryen\ngit checkout tags/1.37.14\ncmake . && make -j 4\nsudo make install\n```\n## 3.4. Compile LLVM WebAssembly+ Host Target\n  - **TRAP 1** \nDue the EOS need LLVM to compile C++ to WebAssembly, the frontend of Clang and WebAssembly are both required, the official compile guidance only compile the WebAssembly target (I assume that is based on the clang is already installed on the ubuntu system,so the auther omitted it when compile LLVM for WebAssembly.),  when we compile the LLVM wtih GNU compiler we need to both compile the WebAssembly and X86 target (which is achived by adding parameter `DLLVM_TARGETS_TO_BUILD='host'` to the cmake configuration), otherwise we would encounter missing  `LLVMX86****.so` error when you compile EOS.\n - **TRAP 2**\nAt current, the EOS only support LLVM of version 4.0,which is achived by clone branch 40 from [llvm mirror site](https://github.com/llvm-mirror/llvm.git), **DO NOT** directlly pull the latest source from github.\n - **TRAP 3**\nYou have to enable RTTI when you compile, otherwise you may encounter error like ***typeof ** undefine*** when you compile EOS. You can avoid this error by add parameter `DLLVM_ENABLE_RTTI=ON` to the cmake configuration.\n\n```bash\ncd  ${BUILD_TEMP}\n\nmkdir ${BUILD_TEMP}/wasm-compiler/build -p\ncd ${BUILD_TEMP}/wasm-compiler\ngit clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/llvm.git\ncd llvm/tools\ngit clone --depth 1 --single-branch --branch release_40 https://github.com/llvm-mirror/clang.git\ncd ${BUILD_TEMP}/wasm-compiler/build\ncmake -G \"Unix Makefiles\" -DLLVM_ENABLE_RTTI=ON -DLLVM_TARGETS_TO_BUILD='host' -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly -DCMAKE_BUILD_TYPE=Release ../llvm\nsudo make -j4 install\n\nexport WASM_LLVM_CONFIG=/usr/local/bin/llvm-config\n```\n# 4. Compile EOS\nFinally, when you completed the all the dependancy compilation, you may proceed to compile the EOS source now.\n```bash\ncd ${BUILD_TEMP}\ngit clone https://github.com/eosio/eos --recursive\nmkdir -p ${BUILD_TEMP}/eos/build && cd ${BUILD_TEMP}/eos/build\ncmake -DBOOST_INCLUDEDIR=${BOOST_ROOT}/include/boost \\\n-DBOOST_LIBRARYDIR=${BOOST_ROOT}/lib  ..\nmake -j4\n```\n# 5. Run your first EOS node!\nPlease refer to [Creating and launching a single-node testnet](https://github.com/EOSIO/eos/wiki/Local-Environment#4-creating-and-launching-a-single-node-testnet).",
      "json_metadata": "{\"tags\":[\"eos\"],\"links\":[\"https://github.com/EOSIO/eos\",\"https://github.com/EOSIO/eos/wiki/Local-Environment#2-building-eos\",\"https://github.com/llvm-mirror/llvm.git\",\"https://github.com/EOSIO/eos/wiki/Local-Environment#4-creating-and-launching-a-single-node-testnet\"],\"app\":\"steemit/0.1\",\"format\":\"markdown\"}",
      "parent_author": "",
      "parent_permlink": "eos",
      "permlink": "how-to-compile-eos-on-clean-centos-system",
      "title": "How to compile EOS  on clean CentOS system"
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T02:04:03",
  "trx_id": "7b21addbc23021976c75d09adc335f252818c6df",
  "trx_in_block": 5,
  "virtual_op": 0
}
steemcreated a new account: @highland0971
2018/02/01 01:13:39
19,475,057|36c8059
active{"account_auths":[],"key_auths":[["STM8V52JgqPnhC8Fo7tsoWPuagrV9T3afu2GFPnVYzTEbwzHxiEZ3",1]],"weight_threshold":1}
creatorsteem
delegation29700.000000 VESTS
extensions[]
fee0.500 STEEM
json metadata
memo keySTM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp
new account namehighland0971
owner{"account_auths":[],"key_auths":[["STM79QY49asjRKnpXEZdapGfjXg18r2EPFsXRxzjoNRo82WNXQ3QE",1]],"weight_threshold":1}
posting{"account_auths":[],"key_auths":[["STM5z3eJEV19juX6NNmUQqYgnebToE7LYk5VJKKgZnGPM8vExbEWD",1]],"weight_threshold":1}
Transaction InfoBlock #19475057/Trx 36c8059c7c4e3cb40213b156c84d348377ccf2f4
View Raw JSON Data
{
  "block": 19475057,
  "op": [
    "account_create_with_delegation",
    {
      "active": {
        "account_auths": [],
        "key_auths": [
          [
            "STM8V52JgqPnhC8Fo7tsoWPuagrV9T3afu2GFPnVYzTEbwzHxiEZ3",
            1
          ]
        ],
        "weight_threshold": 1
      },
      "creator": "steem",
      "delegation": "29700.000000 VESTS",
      "extensions": [],
      "fee": "0.500 STEEM",
      "json_metadata": "",
      "memo_key": "STM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp",
      "new_account_name": "highland0971",
      "owner": {
        "account_auths": [],
        "key_auths": [
          [
            "STM79QY49asjRKnpXEZdapGfjXg18r2EPFsXRxzjoNRo82WNXQ3QE",
            1
          ]
        ],
        "weight_threshold": 1
      },
      "posting": {
        "account_auths": [],
        "key_auths": [
          [
            "STM5z3eJEV19juX6NNmUQqYgnebToE7LYk5VJKKgZnGPM8vExbEWD",
            1
          ]
        ],
        "weight_threshold": 1
      }
    }
  ],
  "op_in_trx": 0,
  "timestamp": "2018-02-01T01:13:39",
  "trx_id": "36c8059c7c4e3cb40213b156c84d348377ccf2f4",
  "trx_in_block": 8,
  "virtual_op": 0
}

Account Metadata

POSTING JSON METADATA
profile{"name":"Highland","website":"https://highland0971.github.io/","profile_image":"https://upload.jianshu.io/users/upload.jianshu.io/users/upload_avatars/4032965/0d1af180631a?imageMogr2/auto-orient/strip|imageView2/1/w/300/h/300"}
JSON METADATA
profile{"name":"Highland","website":"https://highland0971.github.io/","profile_image":"https://upload.jianshu.io/users/upload.jianshu.io/users/upload_avatars/4032965/0d1af180631a?imageMogr2/auto-orient/strip|imageView2/1/w/300/h/300"}
{
  "posting_json_metadata": {
    "profile": {
      "name": "Highland",
      "website": "https://highland0971.github.io/",
      "profile_image": "https://upload.jianshu.io/users/upload.jianshu.io/users/upload_avatars/4032965/0d1af180631a?imageMogr2/auto-orient/strip|imageView2/1/w/300/h/300"
    }
  },
  "json_metadata": {
    "profile": {
      "name": "Highland",
      "website": "https://highland0971.github.io/",
      "profile_image": "https://upload.jianshu.io/users/upload.jianshu.io/users/upload_avatars/4032965/0d1af180631a?imageMogr2/auto-orient/strip|imageView2/1/w/300/h/300"
    }
  }
}

Auth Keys

Owner
Single Signature
Public Keys
STM79QY49asjRKnpXEZdapGfjXg18r2EPFsXRxzjoNRo82WNXQ3QE1/1
Active
Single Signature
Public Keys
STM8V52JgqPnhC8Fo7tsoWPuagrV9T3afu2GFPnVYzTEbwzHxiEZ31/1
Posting
Single Signature
Public Keys
STM5z3eJEV19juX6NNmUQqYgnebToE7LYk5VJKKgZnGPM8vExbEWD1/1
Memo
STM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp
{
  "owner": {
    "account_auths": [],
    "key_auths": [
      [
        "STM79QY49asjRKnpXEZdapGfjXg18r2EPFsXRxzjoNRo82WNXQ3QE",
        1
      ]
    ],
    "weight_threshold": 1
  },
  "active": {
    "account_auths": [],
    "key_auths": [
      [
        "STM8V52JgqPnhC8Fo7tsoWPuagrV9T3afu2GFPnVYzTEbwzHxiEZ3",
        1
      ]
    ],
    "weight_threshold": 1
  },
  "posting": {
    "account_auths": [],
    "key_auths": [
      [
        "STM5z3eJEV19juX6NNmUQqYgnebToE7LYk5VJKKgZnGPM8vExbEWD",
        1
      ]
    ],
    "weight_threshold": 1
  },
  "memo": "STM4xvFyWNsoozCDe7yhEvkofjnr9rj22wcdBeuDX7ADqweq7tzvp"
}

Witness Votes

0 / 30
No active witness votes.
[]