From e4faf64ea3966198f9d7f3fc3a98b6c0e4ef2205 Mon Sep 17 00:00:00 2001 From: MaysWind Date: Fri, 9 Aug 2024 00:07:32 +0800 Subject: [PATCH] redesign the default request id generator, replace random number to client port --- cmd/utility.go | 5 +- pkg/core/context.go | 34 ++++++++++ pkg/middlewares/request_id.go | 2 +- pkg/requestid/default_request_id_generator.go | 66 ++++++++++--------- .../default_request_id_generator_test.go | 59 +++++++++++++++-- pkg/requestid/request_id_container.go | 4 +- pkg/requestid/request_id_generator.go | 2 +- 7 files changed, 128 insertions(+), 44 deletions(-) diff --git a/cmd/utility.go b/cmd/utility.go index c2f4bbd3..fda4b53d 100644 --- a/cmd/utility.go +++ b/cmd/utility.go @@ -66,7 +66,7 @@ func parseRequestId(c *cli.Context) error { return err } - newRequestId := defaultGenerator.GenerateRequestId(net.IPv4zero.String()) + newRequestId := defaultGenerator.GenerateRequestId(net.IPv4zero.String(), 0) newRequestIdInfo, err := defaultGenerator.ParseRequestIdInfo(newRequestId) printRequestIdInfo(requestId, requestIdInfo, newRequestIdInfo) @@ -114,7 +114,6 @@ func printRequestIdInfo(requestId string, requestIdInfo *requestid.RequestIdInfo fmt.Printf("[SecondsElapsedToday] %d\n", requestIdInfo.SecondsElapsedToday) } - fmt.Printf("[RandomNumber] %d\n", requestIdInfo.RandomNumber) fmt.Printf("[RequestSeqId] %d\n", requestIdInfo.RequestSeqId) fmt.Printf("[IsClientIpv6] %t\n", requestIdInfo.IsClientIpv6) @@ -125,4 +124,6 @@ func printRequestIdInfo(requestId string, requestIdInfo *requestid.RequestIdInfo binary.BigEndian.PutUint32(ip, requestIdInfo.ClientIp) fmt.Printf("[ClientIpv4] %s\n", ip.String()) } + + fmt.Printf("[ClientPort] %d\n", requestIdInfo.ClientPort) } diff --git a/pkg/core/context.go b/pkg/core/context.go index 4033251e..2b464fc1 100644 --- a/pkg/core/context.go +++ b/pkg/core/context.go @@ -1,6 +1,7 @@ package core import ( + "net" "strconv" "github.com/gin-gonic/gin" @@ -16,6 +17,9 @@ const responseErrorFieldKey = "RESPONSE_ERROR" // AcceptLanguageHeaderName represents the header name of accept language const AcceptLanguageHeaderName = "Accept-Language" +// RemoteClientPortHeader represents the header name of remote client source port +const RemoteClientPortHeader = "X-Real-Port" + // ClientTimezoneOffsetHeaderName represents the header name of client timezone offset const ClientTimezoneOffsetHeaderName = "X-Timezone-Offset" @@ -25,6 +29,36 @@ type Context struct { // DO NOT ADD ANY FIELD IN THIS CONTEXT, THIS CONTEXT IS JUST A WRAPPER } +func (c *Context) ClientPort() uint16 { + remotePort := c.GetHeader(RemoteClientPortHeader) + + if remotePort != "" { + remotePortNum, err := strconv.ParseInt(remotePort, 10, 32) + + if err == nil { + return uint16(remotePortNum) + } + } + + if c.Request == nil { + return 0 + } + + _, remotePort, err := net.SplitHostPort(c.Request.RemoteAddr) + + if err != nil { + return 0 + } + + remotePortNum, err := strconv.ParseInt(remotePort, 10, 32) + + if err != nil { + return 0 + } + + return uint16(remotePortNum) +} + // SetRequestId sets the given request id to context func (c *Context) SetRequestId(requestId string) { c.Set(requestIdFieldKey, requestId) diff --git a/pkg/middlewares/request_id.go b/pkg/middlewares/request_id.go index 98f825ae..f7e80622 100644 --- a/pkg/middlewares/request_id.go +++ b/pkg/middlewares/request_id.go @@ -16,7 +16,7 @@ func RequestId(config *settings.Config) core.MiddlewareHandlerFunc { return } - requestId := requestid.Container.Current.GenerateRequestId(c.ClientIP()) + requestId := requestid.Container.Current.GenerateRequestId(c.ClientIP(), c.ClientPort()) c.SetRequestId(requestId) if config.EnableRequestIdHeader { diff --git a/pkg/requestid/default_request_id_generator.go b/pkg/requestid/default_request_id_generator.go index 48f82956..bdae4151 100644 --- a/pkg/requestid/default_request_id_generator.go +++ b/pkg/requestid/default_request_id_generator.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "hash/crc32" - "math" "net" "sync/atomic" "time" @@ -19,15 +18,18 @@ import ( // Length and mask of all information in request id const ( - requestIdLength = 36 - secondsTodayBits = 17 - secondsTodayBitsMask = (1 << secondsTodayBits) - 1 - randomNumberBits = 15 - randomNumberBitsMask = (1 << randomNumberBits) - 1 - reqSeqNumberBits = 31 - reqSeqNumberBitsMask = (1 << reqSeqNumberBits) - 1 - clientIpv6Bit = 1 - clientIpv6BitMask = 1 + requestIdLength = 36 + secondsTodayBits = 17 + secondsTodayBitsMask = (1 << secondsTodayBits) - 1 + clientPortNumberAllBits = 16 + clientPortNumberHigh1Bit = 1 + clientPortNumberLow15Bits = clientPortNumberAllBits - clientPortNumberHigh1Bit + clientPortNumberHigh1BitMask = 1 << clientPortNumberLow15Bits + clientPortNumberLow15BitsMask = clientPortNumberHigh1BitMask - 1 + reqSeqNumberBits = 30 + reqSeqNumberBitsMask = (1 << reqSeqNumberBits) - 1 + clientIpv6Bit = 1 + clientIpv6BitMask = 1 ) // RequestIdInfo represents a struct which has all information in request id @@ -35,10 +37,10 @@ type RequestIdInfo struct { ServerUniqId uint16 InstanceUniqId uint16 SecondsElapsedToday uint32 - RandomNumber uint32 RequestSeqId uint32 IsClientIpv6 bool ClientIp uint32 + ClientPort uint16 } // DefaultRequestIdGenerator represents default request id generator @@ -121,7 +123,7 @@ func (r *DefaultRequestIdGenerator) GetCurrentInstanceUniqId() uint16 { } // GenerateRequestId returns a new request id -func (r *DefaultRequestIdGenerator) GenerateRequestId(clientIpAddr string) string { +func (r *DefaultRequestIdGenerator) GenerateRequestId(clientIpAddr string, clientPort uint16) string { ip := net.ParseIP(clientIpAddr) isClientIpv6 := ip.To4() == nil var clientIp uint32 @@ -132,38 +134,38 @@ func (r *DefaultRequestIdGenerator) GenerateRequestId(clientIpAddr string) strin clientIp = binary.BigEndian.Uint32(ip.To4()) } - requestId := r.getRequestId(r.serverUniqId, r.instanceUniqId, isClientIpv6, clientIp) + requestId := r.getRequestId(r.serverUniqId, r.instanceUniqId, isClientIpv6, clientIp, clientPort) return requestId } -func (r *DefaultRequestIdGenerator) getRequestId(serverUniqId uint16, instanceUniqId uint16, clientIpV6 bool, clientIp uint32) string { +func (r *DefaultRequestIdGenerator) getRequestId(serverUniqId uint16, instanceUniqId uint16, clientIpV6 bool, clientIp uint32, clientPort uint16) string { clientIpv6Flag := uint32(0) if clientIpV6 { clientIpv6Flag = uint32(1) } - // 128bits = serverUniqId(16bits) + instanceUniqId(16bits) + secondsElapsedToday(17bits) + randomNumber(15bits) + sequentialNumber(31bits) + clientIpv6Flag(1bit) + clientIp(32bits) + // 128bits = serverUniqId(16bits) + instanceUniqId(16bits) + secondsElapsedToday(17bits) + clientPortLow15Bits(15bits) + sequentialNumber(30bits) + clientPortHigh1Bit(1bit) + clientIpv6Flag(1bit) + clientIp(32bits) secondsElapsedToday := r.getSecondsElapsedToday() secondsLow17bits := uint32(secondsElapsedToday & secondsTodayBitsMask) - randomNumber, _ := utils.GetRandomInteger(math.MaxInt16) - randomNumberLow15bits := uint32(randomNumber & randomNumberBitsMask) + clientPortHigh1bit := uint32((clientPort & clientPortNumberHigh1BitMask) >> clientPortNumberLow15Bits) + clientPortLow15bits := uint32(clientPort & clientPortNumberLow15BitsMask) - secondsAndRandomNumber := (secondsLow17bits << randomNumberBits) | randomNumberLow15bits + secondsAndClientPortLowBits := (secondsLow17bits << clientPortNumberLow15Bits) | clientPortLow15bits seqId := r.requestSeqId.Add(1) - seqIdLow31bits := seqId & reqSeqNumberBitsMask + seqIdLow30bits := seqId & reqSeqNumberBitsMask - seqIdAndClientIpv6Flag := (seqIdLow31bits << clientIpv6Bit) | (clientIpv6Flag & clientIpv6BitMask) + seqIdAndClientPortHighBitAndClientIpv6Flag := (seqIdLow30bits << (clientPortNumberHigh1Bit + clientIpv6Bit)) | (clientPortHigh1bit << clientPortNumberHigh1Bit) | (clientIpv6Flag & clientIpv6BitMask) buf := &bytes.Buffer{} _ = binary.Write(buf, binary.BigEndian, serverUniqId) _ = binary.Write(buf, binary.BigEndian, instanceUniqId) - _ = binary.Write(buf, binary.BigEndian, secondsAndRandomNumber) - _ = binary.Write(buf, binary.BigEndian, seqIdAndClientIpv6Flag) + _ = binary.Write(buf, binary.BigEndian, secondsAndClientPortLowBits) + _ = binary.Write(buf, binary.BigEndian, seqIdAndClientPortHighBitAndClientIpv6Flag) _ = binary.Write(buf, binary.BigEndian, clientIp) return r.getUuidFromRequestId(buf) @@ -198,23 +200,25 @@ func (r *DefaultRequestIdGenerator) parseRequestIdInfo(data []byte) *RequestIdIn var serverUniqId uint16 var instanceUniqId uint16 - var secondsAndRandomNumber uint32 - var seqIdAndClientIpv6Flag uint32 + var secondsAndClientPortLowBits uint32 + var seqIdAndClientPortHighBitAndClientIpv6Flag uint32 var clientIp uint32 _ = binary.Read(buf, binary.BigEndian, &serverUniqId) _ = binary.Read(buf, binary.BigEndian, &instanceUniqId) - _ = binary.Read(buf, binary.BigEndian, &secondsAndRandomNumber) - _ = binary.Read(buf, binary.BigEndian, &seqIdAndClientIpv6Flag) + _ = binary.Read(buf, binary.BigEndian, &secondsAndClientPortLowBits) + _ = binary.Read(buf, binary.BigEndian, &seqIdAndClientPortHighBitAndClientIpv6Flag) _ = binary.Read(buf, binary.BigEndian, &clientIp) - secondsElapsedToday := (secondsAndRandomNumber >> randomNumberBits) & secondsTodayBitsMask - randomNumber := (secondsAndRandomNumber & randomNumberBitsMask) + secondsElapsedToday := (secondsAndClientPortLowBits >> clientPortNumberLow15Bits) & secondsTodayBitsMask - seqId := (seqIdAndClientIpv6Flag >> clientIpv6Bit) & reqSeqNumberBitsMask - isClientIpv6Flag := (seqIdAndClientIpv6Flag & clientIpv6BitMask) + seqId := (seqIdAndClientPortHighBitAndClientIpv6Flag >> (clientPortNumberHigh1Bit + clientIpv6Bit)) & reqSeqNumberBitsMask + clientPortHigh1Bit := ((seqIdAndClientPortHighBitAndClientIpv6Flag >> clientIpv6Bit) << clientPortNumberLow15Bits) & clientPortNumberHigh1BitMask + isClientIpv6Flag := seqIdAndClientPortHighBitAndClientIpv6Flag & clientIpv6BitMask isClientIpv6 := false + clientPort := uint16(clientPortHigh1Bit | (secondsAndClientPortLowBits & clientPortNumberLow15BitsMask)) + if isClientIpv6Flag == 1 { isClientIpv6 = true } @@ -224,9 +228,9 @@ func (r *DefaultRequestIdGenerator) parseRequestIdInfo(data []byte) *RequestIdIn InstanceUniqId: instanceUniqId, SecondsElapsedToday: secondsElapsedToday, RequestSeqId: seqId, - RandomNumber: randomNumber, IsClientIpv6: isClientIpv6, ClientIp: clientIp, + ClientPort: clientPort, } } diff --git a/pkg/requestid/default_request_id_generator_test.go b/pkg/requestid/default_request_id_generator_test.go index 70ec1dfb..7090a8dd 100644 --- a/pkg/requestid/default_request_id_generator_test.go +++ b/pkg/requestid/default_request_id_generator_test.go @@ -10,7 +10,7 @@ import ( func TestNewDefaultRequestIdGenerator_Http(t *testing.T) { generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "123.234.123.234", HttpPort: 8080, SecretKey: "secretkey"}) - requestId := generator.GenerateRequestId("127.0.0.1") + requestId := generator.GenerateRequestId("127.0.0.1", 20000) requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) expectedServerUniqId := uint16(0x2476) // crc32("123.234.123.234" + "_" + "secretkey") & 0xFFFF @@ -24,7 +24,7 @@ func TestNewDefaultRequestIdGenerator_Http(t *testing.T) { func TestNewDefaultRequestIdGenerator_UnixSocket(t *testing.T) { generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", UnixSocketPath: "/var/lib/ezbookkeeping/ezbookkeeping.sock", Protocol: "socket", SecretKey: "secretkey"}) - requestId := generator.GenerateRequestId("127.0.0.1") + requestId := generator.GenerateRequestId("127.0.0.1", 20000) requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) expectedServerUniqId := uint16(0x5bdb) // crc32("1.2.3.4" + "_" + "secretkey") & 0xFFFF @@ -38,7 +38,7 @@ func TestNewDefaultRequestIdGenerator_UnixSocket(t *testing.T) { func TestNewDefaultRequestIdGenerator_ClientIpv4(t *testing.T) { generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", UnixSocketPath: "/var/lib/ezbookkeeping/ezbookkeeping.sock", Protocol: "socket", SecretKey: "secretkey"}) - requestId := generator.GenerateRequestId("127.0.0.1") + requestId := generator.GenerateRequestId("127.0.0.1", 20000) requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) expectedClientIp := uint32(0x7f000001) // 127.0.0.1 @@ -49,7 +49,7 @@ func TestNewDefaultRequestIdGenerator_ClientIpv4(t *testing.T) { actualClientIpv6 := requestIdInfo.IsClientIpv6 assert.Equal(t, expectedClientIpv6, actualClientIpv6) - requestId = generator.GenerateRequestId("192.168.1.100") + requestId = generator.GenerateRequestId("192.168.1.100", 20000) requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) expectedClientIp = uint32(0xc0a80164) // 192.168.1.100 @@ -63,7 +63,7 @@ func TestNewDefaultRequestIdGenerator_ClientIpv4(t *testing.T) { func TestNewDefaultRequestIdGenerator_ClientIpv6(t *testing.T) { generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", UnixSocketPath: "/var/lib/ezbookkeeping/ezbookkeeping.sock", Protocol: "socket", SecretKey: "secretkey"}) - requestId := generator.GenerateRequestId("2001:abc:def:1234::1") + requestId := generator.GenerateRequestId("2001:abc:def:1234::1", 20000) requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) expectedClientIp := uint32(0x76fe1b98) // crc32("2001:abc:def:1234::1") @@ -74,7 +74,7 @@ func TestNewDefaultRequestIdGenerator_ClientIpv6(t *testing.T) { actualClientIpv6 := requestIdInfo.IsClientIpv6 assert.Equal(t, expectedClientIpv6, actualClientIpv6) - requestId = generator.GenerateRequestId("2400:abcd:1234:1:56ef:ab78:c90d:1e2f") + requestId = generator.GenerateRequestId("2400:abcd:1234:1:56ef:ab78:c90d:1e2f", 20000) requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) expectedClientIp = uint32(0xa0a25faa) // crc32("2400:abcd:1234:1:56ef:ab78:c90d:1e2f") @@ -86,11 +86,56 @@ func TestNewDefaultRequestIdGenerator_ClientIpv6(t *testing.T) { assert.Equal(t, expectedClientIpv6, actualClientIpv6) } +func TestNewDefaultRequestIdGenerator_ClientPort(t *testing.T) { + generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", UnixSocketPath: "/var/lib/ezbookkeeping/ezbookkeeping.sock", Protocol: "socket", SecretKey: "secretkey"}) + requestId := generator.GenerateRequestId("127.0.0.1", 0) + requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) + + expectedClientPort := uint16(0) + actualClientPort := requestIdInfo.ClientPort + assert.Equal(t, expectedClientPort, actualClientPort) + + requestId = generator.GenerateRequestId("127.0.0.1", 12345) + requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) + + expectedClientPort = uint16(12345) + actualClientPort = requestIdInfo.ClientPort + assert.Equal(t, expectedClientPort, actualClientPort) + + requestId = generator.GenerateRequestId("127.0.0.1", 32767) + requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) + + expectedClientPort = uint16(32767) + actualClientPort = requestIdInfo.ClientPort + assert.Equal(t, expectedClientPort, actualClientPort) + + requestId = generator.GenerateRequestId("127.0.0.1", 32768) + requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) + + expectedClientPort = uint16(32768) + actualClientPort = requestIdInfo.ClientPort + assert.Equal(t, expectedClientPort, actualClientPort) + + requestId = generator.GenerateRequestId("127.0.0.1", 56789) + requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) + + expectedClientPort = uint16(56789) + actualClientPort = requestIdInfo.ClientPort + assert.Equal(t, expectedClientPort, actualClientPort) + + requestId = generator.GenerateRequestId("127.0.0.1", 65535) + requestIdInfo = generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) + + expectedClientPort = uint16(65535) + actualClientPort = requestIdInfo.ClientPort + assert.Equal(t, expectedClientPort, actualClientPort) +} + func TestGenerateRequestId_100Times(t *testing.T) { generator, _ := NewDefaultRequestIdGenerator(&settings.Config{HttpAddr: "1.2.3.4", HttpPort: 1234, SecretKey: "secretkey"}) for i := 1; i <= 100; i++ { - requestId := generator.GenerateRequestId("127.0.0.1") + requestId := generator.GenerateRequestId("127.0.0.1", 20000) requestIdInfo := generator.parseRequestIdInfo(generator.parseRequestIdFromUuid(requestId)) expectedRequestSeqId := uint32(i) diff --git a/pkg/requestid/request_id_container.go b/pkg/requestid/request_id_container.go index 3ea0b4b0..b285b842 100644 --- a/pkg/requestid/request_id_container.go +++ b/pkg/requestid/request_id_container.go @@ -27,6 +27,6 @@ func InitializeRequestIdGenerator(config *settings.Config) error { } // GenerateRequestId returns a new request id by the current request id generator -func (u *RequestIdContainer) GenerateRequestId(clientIpAddr string) string { - return u.Current.GenerateRequestId(clientIpAddr) +func (u *RequestIdContainer) GenerateRequestId(clientIpAddr string, clientPort uint16) string { + return u.Current.GenerateRequestId(clientIpAddr, clientPort) } diff --git a/pkg/requestid/request_id_generator.go b/pkg/requestid/request_id_generator.go index b57b598d..4698eadf 100644 --- a/pkg/requestid/request_id_generator.go +++ b/pkg/requestid/request_id_generator.go @@ -2,7 +2,7 @@ package requestid // RequestIdGenerator is common request generator interface type RequestIdGenerator interface { - GenerateRequestId(clientIpAddr string) string + GenerateRequestId(clientIpAddr string, clientPort uint16) string GetCurrentServerUniqId() uint16 GetCurrentInstanceUniqId() uint16 }