zhengchen.tao/ezbookkeeping
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

redesign the default request id generator, replace random number to client port

Browse Source
This commit is contained in:
MaysWind
2024-08-09 00:07:32 +08:00
parent a4849fa4f0
commit e4faf64ea3
7 changed files with 128 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/utility.go
+3 -2
View File
@@ -66,7 +66,7 @@ func parseRequestId(c *cli.Context) error {
return err
}
newRequestId := defaultGenerator.GenerateRequestId(net.IPv4zero.String())
newRequestId := defaultGenerator.GenerateRequestId(net.IPv4zero.String(), 0)
newRequestIdInfo, err := defaultGenerator.ParseRequestIdInfo(newRequestId)
printRequestIdInfo(requestId, requestIdInfo, newRequestIdInfo)
@@ -114,7 +114,6 @@ func printRequestIdInfo(requestId string, requestIdInfo *requestid.RequestIdInfo
fmt.Printf("[SecondsElapsedToday] %d\n", requestIdInfo.SecondsElapsedToday)
}
fmt.Printf("[RandomNumber] %d\n", requestIdInfo.RandomNumber)
fmt.Printf("[RequestSeqId] %d\n", requestIdInfo.RequestSeqId)
fmt.Printf("[IsClientIpv6] %t\n", requestIdInfo.IsClientIpv6)
@@ -125,4 +124,6 @@ func printRequestIdInfo(requestId string, requestIdInfo *requestid.RequestIdInfo
binary.BigEndian.PutUint32(ip, requestIdInfo.ClientIp)
fmt.Printf("[ClientIpv4] %s\n", ip.String())
}
fmt.Printf("[ClientPort] %d\n", requestIdInfo.ClientPort)
}
pkg/core/context.go
+34
View File
@@ -1,6 +1,7 @@
package core
import (
"net"
"strconv"
"github.com/gin-gonic/gin"
@@ -16,6 +17,9 @@ const responseErrorFieldKey = "RESPONSE_ERROR"
// AcceptLanguageHeaderName represents the header name of accept language
const AcceptLanguageHeaderName = "Accept-Language"
// RemoteClientPortHeader represents the header name of remote client source port
const RemoteClientPortHeader = "X-Real-Port"
// ClientTimezoneOffsetHeaderName represents the header name of client timezone offset
const ClientTimezoneOffsetHeaderName = "X-Timezone-Offset"
@@ -25,6 +29,36 @@ type Context struct {
// DO NOT ADD ANY FIELD IN THIS CONTEXT, THIS CONTEXT IS JUST A WRAPPER
}
func (c *Context) ClientPort() uint16 {
remotePort := c.GetHeader(RemoteClientPortHeader)
if remotePort != "" {
remotePortNum, err := strconv.ParseInt(remotePort, 10, 32)
if err == nil {
return uint16(remotePortNum)
}
}
if c.Request == nil {
return 0
}
_, remotePort, err := net.SplitHostPort(c.Request.RemoteAddr)
if err != nil {
return 0
}
remotePortNum, err := strconv.ParseInt(remotePort, 10, 32)
if err != nil {
return 0
}
return uint16(remotePortNum)
}
// SetRequestId sets the given request id to context
func (c *Context) SetRequestId(requestId string) {
c.Set(requestIdFieldKey, requestId)
pkg/middlewares/request_id.go
+1 -1
View File
@@ -16,7 +16,7 @@ func RequestId(config *settings.Config) core.MiddlewareHandlerFunc {
return
}
requestId := requestid.Container.Current.GenerateRequestId(c.ClientIP())
requestId := requestid.Container.Current.GenerateRequestId(c.ClientIP(), c.ClientPort())
c.SetRequestId(requestId)
if config.EnableRequestIdHeader {
pkg/requestid/default_request_id_generator.go
+29 -25
View File
@@ -6,7 +6,6 @@ import (
"encoding/hex"
"fmt"
"hash/crc32"
"math"
"net"
"sync/atomic"
"time"
@@ -22,9 +21,12 @@ const (
requestIdLength = 36
secondsTodayBits = 17
secondsTodayBitsMask = (1 << secondsTodayBits) - 1
randomNumberBits = 15
randomNumberBitsMask = (1 << randomNumberBits) - 1
reqSeqNumberBits = 31
clientPortNumberAllBits = 16
clientPortNumberHigh1Bit = 1
clientPortNumberLow15Bits = clientPortNumberAllBits - clientPortNumberHigh1Bit
clientPortNumberHigh1BitMask = 1 << clientPortNumberLow15Bits
clientPortNumberLow15BitsMask = clientPortNumberHigh1BitMask - 1
reqSeqNumberBits = 30
reqSeqNumberBitsMask = (1 << reqSeqNumberBits) - 1
clientIpv6Bit = 1
clientIpv6BitMask = 1
@@ -35,10 +37,10 @@ type RequestIdInfo struct {
ServerUniqId uint16
InstanceUniqId uint16
SecondsElapsedToday uint32
RandomNumber uint32
RequestSeqId uint32
IsClientIpv6 bool
ClientIp uint32
ClientPort uint16
}
// DefaultRequestIdGenerator represents default request id generator
@@ -121,7 +123,7 @@ func (r *DefaultRequestIdGenerator) GetCurrentInstanceUniqId() uint16 {
}
// GenerateRequestId returns a new request id
func (r *DefaultRequestIdGenerator) GenerateRequestId(clientIpAddr string) string {
func (r *DefaultRequestIdGenerator) GenerateRequestId(clientIpAddr string, clientPort uint16) string {
ip := net.ParseIP(clientIpAddr)
isClientIpv6 := ip.To4() == nil
var clientIp uint32
@@ -132,38 +134,38 @@ func (r *DefaultRequestIdGenerator) GenerateRequestId(clientIpAddr string) strin
clientIp = binary.BigEndian.Uint32(ip.To4())
}
requestId := r.getRequestId(r.serverUniqId, r.instanceUniqId, isClientIpv6, clientIp)
requestId := r.getRequestId(r.serverUniqId, r.instanceUniqId, isClientIpv6, clientIp, clientPort)
return requestId
}
func (r *DefaultRequestIdGenerator) getRequestId(serverUniqId uint16, instanceUniqId uint16, clientIpV6 bool, clientIp uint32) string {
func (r *DefaultRequestIdGenerator) getRequestId(serverUniqId uint16, instanceUniqId uint16, clientIpV6 bool, clientIp uint32, clientPort uint16) string {
clientIpv6Flag := uint32(0)
if clientIpV6 {
clientIpv6Flag = uint32(1)
}
// 128bits = serverUniqId(16bits) + instanceUniqId(16bits) + secondsElapsedToday(17bits) + randomNumber(15bits) + sequentialNumber(31bits) + clientIpv6Flag(1bit) + clientIp(32bits)
// 128bits = serverUniqId(16bits) + instanceUniqId(16bits) + secondsElapsedToday(17bits) + clientPortLow15Bits(15bits) + sequentialNumber(30bits) + clientPortHigh1Bit(1bit) + clientIpv6Flag(1bit) + clientIp(32bits)
secondsElapsedToday := r.getSecondsElapsedToday()
secondsLow17bits := uint32(secondsElapsedToday & secondsTodayBitsMask)
randomNumber, _ := utils.GetRandomInteger(math.MaxInt16)
randomNumberLow15bits := uint32(randomNumber & randomNumberBitsMask)
clientPortHigh1bit := uint32((clientPort & clientPortNumberHigh1BitMask) >> clientPortNumberLow15Bits)
clientPortLow15bits := uint32(clientPort & clientPortNumberLow15BitsMask)
secondsAndRandomNumber := (secondsLow17bits << randomNumberBits) | randomNumberLow15bits
secondsAndClientPortLowBits := (secondsLow17bits << clientPortNumberLow15Bits) | clientPortLow15bits
seqId := r.requestSeqId.Add(1)
seqIdLow31bits := seqId & reqSeqNumberBitsMask
seqIdLow30bits := seqId & reqSeqNumberBitsMask
seqIdAndClientIpv6Flag := (seqIdLow31bits << clientIpv6Bit) | (clientIpv6Flag & clientIpv6BitMask)
seqIdAndClientPortHighBitAndClientIpv6Flag := (seqIdLow30bits << (clientPortNumberHigh1Bit + clientIpv6Bit)) | (clientPortHigh1bit << clientPortNumberHigh1Bit) | (clientIpv6Flag & clientIpv6BitMask)
buf := &bytes.Buffer{}
_ = binary.Write(buf, binary.BigEndian, serverUniqId)
_ = binary.Write(buf, binary.BigEndian, instanceUniqId)
_ = binary.Write(buf, binary.BigEndian, secondsAndRandomNumber)
_ = binary.Write(buf, binary.BigEndian, seqIdAndClientIpv6Flag)
_ = binary.Write(buf, binary.BigEndian, secondsAndClientPortLowBits)
_ = binary.Write(buf, binary.BigEndian, seqIdAndClientPortHighBitAndClientIpv6Flag)
_ = binary.Write(buf, binary.BigEndian, clientIp)
return r.getUuidFromRequestId(buf)
@@ -198,23 +200,25 @@ func (r *DefaultRequestIdGenerator) parseRequestIdInfo(data []byte) *RequestIdIn
var serverUniqId uint16
var instanceUniqId uint16
var secondsAndRandomNumber uint32
var seqIdAndClientIpv6Flag uint32
var secondsAndClientPortLowBits uint32
var seqIdAndClientPortHighBitAndClientIpv6Flag uint32
var clientIp uint32
_ = binary.Read(buf, binary.BigEndian, &serverUniqId)
_ = binary.Read(buf, binary.BigEndian, &instanceUniqId)
_ = binary.Read(buf, binary.BigEndian, &secondsAndRandomNumber)
_ = binary.Read(buf, binary.BigEndian, &seqIdAndClientIpv6Flag)
_ = binary.Read(buf, binary.BigEndian, &secondsAndClientPortLowBits)
_ = binary.Read(buf, binary.BigEndian, &seqIdAndClientPortHighBitAndClientIpv6Flag)
_ = binary.Read(buf, binary.BigEndian, &clientIp)
secondsElapsedToday := (secondsAndRandomNumber >> randomNumberBits) & secondsTodayBitsMask
randomNumber := (secondsAndRandomNumber & randomNumberBitsMask)
secondsElapsedToday := (secondsAndClientPortLowBits >> clientPortNumberLow15Bits) & secondsTodayBitsMask
seqId := (seqIdAndClientIpv6Flag >> clientIpv6Bit) & reqSeqNumberBitsMask
isClientIpv6Flag := (seqIdAndClientIpv6Flag & clientIpv6BitMask)
seqId := (seqIdAndClientPortHighBitAndClientIpv6Flag >> (clientPortNumberHigh1Bit + clientIpv6Bit)) & reqSeqNumberBitsMask
clientPortHigh1Bit := ((seqIdAndClientPortHighBitAndClientIpv6Flag >> clientIpv6Bit) << clientPortNumberLow15Bits) & clientPortNumberHigh1BitMask
isClientIpv6Flag := seqIdAndClientPortHighBitAndClientIpv6Flag & clientIpv6BitMask
isClientIpv6 := false
clientPort := uint16(clientPortHigh1Bit | (secondsAndClientPortLowBits & clientPortNumberLow15BitsMask))
if isClientIpv6Flag == 1 {
isClientIpv6 = true
}
@@ -224,9 +228,9 @@ func (r *DefaultRequestIdGenerator) parseRequestIdInfo(data []byte) *RequestIdIn
InstanceUniqId: instanceUniqId,
SecondsElapsedToday: secondsElapsedToday,
RequestSeqId: seqId,
RandomNumber: randomNumber,
IsClientIpv6: isClientIpv6,
ClientIp: clientIp,
ClientPort: clientPort,
}
}
pkg/requestid/default_request_id_generator_test.go
+52 -7
View File
@@ -10,7 +10,7 @@ import (
func TestNewDefaultRequestIdGenerator_Http(t *testing.T) {
generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "123.234.123.234", HttpPort: 8080, SecretKey: "secretkey"})
requestId := generator.GenerateRequestId("127.0.0.1")
requestId := generator.GenerateRequestId("127.0.0.1", 20000)
requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedServerUniqId := uint16(0x2476) // crc32("123.234.123.234" + "_" + "secretkey") & 0xFFFF
@@ -24,7 +24,7 @@ func TestNewDefaultRequestIdGenerator_Http(t *testing.T) {
func TestNewDefaultRequestIdGenerator_UnixSocket(t *testing.T) {
generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", UnixSocketPath: "/var/lib/ezbookkeeping/ezbookkeeping.sock", Protocol: "socket", SecretKey: "secretkey"})
requestId := generator.GenerateRequestId("127.0.0.1")
requestId := generator.GenerateRequestId("127.0.0.1", 20000)
requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedServerUniqId := uint16(0x5bdb) // crc32("1.2.3.4" + "_" + "secretkey") & 0xFFFF
@@ -38,7 +38,7 @@ func TestNewDefaultRequestIdGenerator_UnixSocket(t *testing.T) {
func TestNewDefaultRequestIdGenerator_ClientIpv4(t *testing.T) {
generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", UnixSocketPath: "/var/lib/ezbookkeeping/ezbookkeeping.sock", Protocol: "socket", SecretKey: "secretkey"})
requestId := generator.GenerateRequestId("127.0.0.1")
requestId := generator.GenerateRequestId("127.0.0.1", 20000)
requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientIp := uint32(0x7f000001) // 127.0.0.1
@@ -49,7 +49,7 @@ func TestNewDefaultRequestIdGenerator_ClientIpv4(t *testing.T) {
actualClientIpv6 := requestIdInfo.IsClientIpv6
assert.Equal(t, expectedClientIpv6, actualClientIpv6)
requestId = generator.GenerateRequestId("192.168.1.100")
requestId = generator.GenerateRequestId("192.168.1.100", 20000)
requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientIp = uint32(0xc0a80164) // 192.168.1.100
@@ -63,7 +63,7 @@ func TestNewDefaultRequestIdGenerator_ClientIpv4(t *testing.T) {
func TestNewDefaultRequestIdGenerator_ClientIpv6(t *testing.T) {
generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", UnixSocketPath: "/var/lib/ezbookkeeping/ezbookkeeping.sock", Protocol: "socket", SecretKey: "secretkey"})
requestId := generator.GenerateRequestId("2001:abc:def:1234::1")
requestId := generator.GenerateRequestId("2001:abc:def:1234::1", 20000)
requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientIp := uint32(0x76fe1b98) // crc32("2001:abc:def:1234::1")
@@ -74,7 +74,7 @@ func TestNewDefaultRequestIdGenerator_ClientIpv6(t *testing.T) {
actualClientIpv6 := requestIdInfo.IsClientIpv6
assert.Equal(t, expectedClientIpv6, actualClientIpv6)
requestId = generator.GenerateRequestId("2400:abcd:1234:1:56ef:ab78:c90d:1e2f")
requestId = generator.GenerateRequestId("2400:abcd:1234:1:56ef:ab78:c90d:1e2f", 20000)
requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientIp = uint32(0xa0a25faa) // crc32("2400:abcd:1234:1:56ef:ab78:c90d:1e2f")
@@ -86,11 +86,56 @@ func TestNewDefaultRequestIdGenerator_ClientIpv6(t *testing.T) {
assert.Equal(t, expectedClientIpv6, actualClientIpv6)
}
func TestNewDefaultRequestIdGenerator_ClientPort(t *testing.T) {
generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", UnixSocketPath: "/var/lib/ezbookkeeping/ezbookkeeping.sock", Protocol: "socket", SecretKey: "secretkey"})
requestId := generator.GenerateRequestId("127.0.0.1", 0)
requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientPort := uint16(0)
actualClientPort := requestIdInfo.ClientPort
assert.Equal(t, expectedClientPort, actualClientPort)
requestId = generator.GenerateRequestId("127.0.0.1", 12345)
requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientPort = uint16(12345)
actualClientPort = requestIdInfo.ClientPort
assert.Equal(t, expectedClientPort, actualClientPort)
requestId = generator.GenerateRequestId("127.0.0.1", 32767)
requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientPort = uint16(32767)
actualClientPort = requestIdInfo.ClientPort
assert.Equal(t, expectedClientPort, actualClientPort)
requestId = generator.GenerateRequestId("127.0.0.1", 32768)
requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientPort = uint16(32768)
actualClientPort = requestIdInfo.ClientPort
assert.Equal(t, expectedClientPort, actualClientPort)
requestId = generator.GenerateRequestId("127.0.0.1", 56789)
requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientPort = uint16(56789)
actualClientPort = requestIdInfo.ClientPort
assert.Equal(t, expectedClientPort, actualClientPort)
requestId = generator.GenerateRequestId("127.0.0.1", 65535)
requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedClientPort = uint16(65535)
actualClientPort = requestIdInfo.ClientPort
assert.Equal(t, expectedClientPort, actualClientPort)
}
func TestGenerateRequestId_100Times(t *testing.T) {
generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", HttpPort: 1234, SecretKey: "secretkey"})
for i := 1; i <= 100; i++ {
requestId := generator.GenerateRequestId("127.0.0.1")
requestId := generator.GenerateRequestId("127.0.0.1", 20000)
requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId))
expectedRequestSeqId := uint32(i)
pkg/requestid/request_id_container.go
+2 -2
View File
@@ -27,6 +27,6 @@ func InitializeRequestIdGenerator(config *settings.Config) error {
}
// GenerateRequestId returns a new request id by the current request id generator
func (u *RequestIdContainer) GenerateRequestId(clientIpAddr string) string {
return u.Current.GenerateRequestId(clientIpAddr)
func (u *RequestIdContainer) GenerateRequestId(clientIpAddr string, clientPort uint16) string {
return u.Current.GenerateRequestId(clientIpAddr, clientPort)
}
pkg/requestid/request_id_generator.go
+1 -1
View File
@@ -2,7 +2,7 @@ package requestid
// RequestIdGenerator is common request generator interface
type RequestIdGenerator interface {
GenerateRequestId(clientIpAddr string) string
GenerateRequestId(clientIpAddr string, clientPort uint16) string
GetCurrentServerUniqId() uint16
GetCurrentInstanceUniqId() uint16
}