IP to Geo

IP to Geo Commit Details

Date:2016-02-06 14:24:26 (4 years 8 months ago)
Author:Grégory Soutadé
Branch:master
Commit:4b791be9f6806b666c64b75d1aea6662c1271a25
Parents: 401a2e98af356f2a0e07db6caae987a49ae1270c
Message:Add full IPV6 support

Changes:
Mdata/Makefile (2 diffs)
Mdata/build_c_array.py (7 diffs)
Msrc/ip_to_geo.c (1 diff)

File differences

data/Makefile
11
22
3
3
44
55
66
......
2121
2222
2323
24
24
2525
2626
2727
IP_DATA = ../src/ip_data.c
PROVIDERS = afrinic arin apnic lacnic ripencc
DEST = prefix_res_ipv4
DEST = prefix_res
MD5 = $(addsuffix .md5,$(PROVIDERS))
all: clean_md5 $(MD5) $(IP_DATA)
@md5sum -c $@ || wget "ftp://ftp.ripe.net/pub/stats/$(basename $@)/delegated-$(basename $@)-extended-latest" -O $(basename $@) ; true
$(DEST): $(PROVIDERS)
@cat $(PROVIDERS) | grep -v asn | grep -v summary | grep -v '#' | grep ipv4 | sort -n -k 4 -t '|' > $(DEST)
@cat $(PROVIDERS) | grep -v asn | grep -v summary | grep -v '#' | grep ipv[46] | sort -n -k 4 -t '|' > $(DEST)
$(IP_DATA): $(DEST)
@echo "Rebuild ip_data.c"
data/build_c_array.py
88
99
1010
11
11
1212
1313
1414
......
2222
2323
2424
25
26
25
26
2727
28
29
30
31
32
33
34
35
36
28
29
3730
38
39
40
41
42
43
44
45
46
47
48
49
31
32
33
34
35
5036
51
52
53
54
55
56
37
38
39
40
5741
5842
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
5958
6059
6160
......
7170
7271
7372
74
75
76
7773
7874
7975
......
8581
8682
8783
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
88149
89
90
91
92150
93
94
151
152
153
95154
96155
97156
......
108167
109168
110169
111
112
113
114
115
116
117
118
170
171
172
173
174
175
176
119177
120178
121179
122180
123181
124
182
125183
126184
127185
128186
129
130
131
132187
133188
134189
......
183238
184239
185240
186
187
188
189
241
190242
191243
192244
......
215267
216268
217269
218
219
220
221
222
223
224270
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
247304
248305
249306
IP_INDEX=3
IP_SIZE_INDEX=4
class IP_ELEMENT(object):
class IP_ELEMENT(object):
def __init__(self, start, end=None, size=0, country_code=None, level=0):
self._start = start
self._end = end
if not self._end: self._compute_last_ip()
self._splitted_start = IP_ELEMENT.split_ip(self._start)
self._splitted_end = IP_ELEMENT.split_ip(self._end)
self._splitted_start = self.split_ip(self._start)
self._splitted_end = self.split_ip(self._end)
def _compute_last_ip(self):
size = self._size
end_ip = IP_ELEMENT.ip_to_int(self._start)
for i in range(0,4):
if not size: break # We can have _size == 0
end_ip += (((size % 256)-1) & 0xFF) << (i*8)
size = int(size/256)
self._end = IP_ELEMENT.ip_to_str(end_ip)
# print '%s + %d -> %s' % (self._start, self._size, self._end)
def split_ip(self, ip):
return [int(x, self._base) for x in ip.split(self._separator)]
@staticmethod
def split_ip(ip):
return [int(x) for x in ip.split('.')]
@staticmethod
def ip_to_int(str_ip):
splitted_ip = IP_ELEMENT.split_ip(str_ip)
val = splitted_ip[0] << 24
val += splitted_ip[1] << 16
val += splitted_ip[2] << 8
val += splitted_ip[3] << 0
return val
def ip_to_str(self, int_ip):
res = []
for i in range(0, self.get_ip_len()):
res.insert(0, self._format % int((int_ip >> (i*8)) & 0xFF))
return self._separator.join(res)
@staticmethod
def ip_to_str(int_ip):
val = '%d.' % (int((int_ip >> 24) & 0xFF))
val += '%d.' % (int((int_ip >> 16) & 0xFF))
val += '%d.' % (int((int_ip >> 8) & 0xFF))
val += '%d' % (int((int_ip >> 0) & 0xFF))
def ip_array_to_int(self, array):
val = 0
for i in range(0, len(array)):
val += array[len(array)-i-1] << (i*8)
return val
def ip_to_int(self, str_ip):
return self.ip_array_to_int(self.split_ip(str_ip))
def make_group(self):
ip_val = self._splitted_start[::]
for i in range(self._level+1, self.get_ip_len()):
ip_val[i] = 0
return self._separator.join([self._format % x for x in ip_val])
def name(self):
return 'ip__%s__%s' %(self._start.replace(self._separator, '_'), self._end.replace(self._separator, '_'))
def _compute_last_ip(self):
raise NotImplementedError()
def set_next(self, ip):
self._next = ip
def set_level(self, level):
self._level = level
def name(self):
return 'ip__%s__%s' %(self._start.replace('.', '_'), self._end.replace('.', '_'))
def printme(self):
print 'static const ip_level %s = {' % (self.name())
print '\t.prev = %s,' % (self._prev and '&%s' % (self._prev.name()) or 'NULL')
print '\t.code = %d,' % (self._country_code and self._country_code or 0)
print '};'
def get_ip_len(self):
raise NotImplementedError()
class IP_ELEMENT4(IP_ELEMENT):
def __init__(self, start, end=None, size=0, country_code=None, level=0):
self._separator = '.'
self._base = 10
self._format = '%d'
super(IP_ELEMENT4, self).__init__(start, end, size, country_code, level)
def get_ip_len(self):
return 4
def _compute_last_ip(self):
size = self._size
end_ip = self.ip_to_int(self._start)
i=0
while size > 0:
end_ip += (((size % 256)-1) & 0xFF) << (i*8)
size = int(size/256)
i += 1
self._end = self.ip_to_str(end_ip)
# print '%s + %d -> %s' % (self._start, self._size, self._end)
class IP_ELEMENT6(IP_ELEMENT):
def __init__(self, start, end=None, size=0, country_code=None, level=0):
self._separator = ':'
self._base = 16
self._format = '%02x'
super(IP_ELEMENT6, self).__init__(start, end, size, country_code, level)
def get_ip_len(self):
return 16
def _get_mask(self):
mask = 0
for i in range(0, self._size):
mask += 1 << i
mask <<= 128-self._size
return mask
def _compute_last_ip(self):
if self._size == 0:
self._end = self._start[:]
else:
mask = self._get_mask()
self._end = self.ip_to_str(self.ip_to_int(self._start) | ~mask)
def extend_ipv6(ipv6):
tmp = ''
for s in ipv6.split(':'):
if not s: break
while len(s) != 4:
s = '0' + s
tmp += s
while len(tmp) < 16*2:
tmp += '0'
res = ''
for i in range(0, 15*2, 2):
res += tmp[i] + tmp[i+1] + ':'
res += tmp[30] + tmp[31]
return res
countries = []
ip_idx = [0] * 255
cur_ip_prefix = 1
cur_idx = 0
f = open("prefix_res_ipv4")
array_vals = {}
f = open("prefix_res")
array_vals_ipv4 = {}
array_vals_ipv6 = {}
while True:
l = f.readline()
# l = sys.stdin.readline()
countries.append(country)
ip = information[IP_INDEX]
splitted_ip = ip.split('.')
int_ip = int(splitted_ip[0]) << 24
int_ip += int(splitted_ip[1]) << 16
int_ip += int(splitted_ip[2]) << 8
int_ip += int(splitted_ip[3]) << 0
interval_size = int(information[IP_SIZE_INDEX])
array_vals[ip] = IP_ELEMENT(ip, None, interval_size, country_idx)
if information[IP_TYPE_INDEX] == 'ipv4':
array_vals_ipv4[ip] = IP_ELEMENT4(ip, None, int(information[IP_SIZE_INDEX]), country_idx)
elif information[IP_TYPE_INDEX] == 'ipv6':
ip = extend_ipv6(ip)
array_vals_ipv6[ip] = IP_ELEMENT6(ip, None, int(information[IP_SIZE_INDEX]), country_idx)
else:
sys.stderr.write('Unknown IP type %s\n' % (information[IP_TYPE_INDEX]))
print '/* This file was automatically generated, do not edit it ! */'
print '#include <stdint.h>\n\n'
def ip_sort(a, b):
for i in range(0, 4):
for i in range(0, len(a._splitted_start)):
if a._splitted_start[i] != b._splitted_start[i]:
return a._splitted_start[i] - b._splitted_start[i]
return 0
ip_list = array_vals.values()
ip_list.sort(ip_sort)
def get_interval(root, intervals, level):
new_intervals = []
for ip in intervals:
cur_ip.set_level(level+1)
for ip in sub_interval:
ip.set_level(level+1)
ip_val = IP_ELEMENT.ip_to_int(cur_ip._start)
for i in range(level, 3):
ip_val &= ~(0xFF << ((2-i)*8)) & 0xFFFFFFFF
new_group = IP_ELEMENT(IP_ELEMENT.ip_to_str(ip_val), level=level)
new_group = cur_ip.__class__(cur_ip.make_group(), level=level)
sub_interval.insert(0, cur_ip)
child = manage_root(cur_ip._splitted_start[level+1], sub_interval, level+1)
new_group.set_childs(cur_ip)
while cur_ip:
cur_ip.printme()
cur_ip = cur_ip._next
start_idx = 0
end_idx = start_idx+1
cur_interval = [ip_list[start_idx]]
root = ip_list[start_idx]._splitted_start[0]
root_ips = [None] * 256
while True:
if end_idx >= len(ip_list): break
if ip_list[end_idx]._splitted_start[0] != root:
start_idx = end_idx
res = manage_root(root, cur_interval, 1)
print_ip(res)
root_ips[res._splitted_start[0]] = res
cur_interval = [ip_list[end_idx]]
root = ip_list[start_idx]._splitted_start[0]
else:
cur_interval.append(ip_list[end_idx])
end_idx += 1
res = manage_root(root, cur_interval, 1)
print_ip(res)
print 'static const ip_level* s_root_ip[256] = {'
for i in range(0, 256):
if root_ips[i]:
print '\t&%s,' % (root_ips[i].name())
else:
print '\tNULL, // %d' % (i)
print '};\n'
def build_array(ip_list, array_name):
ip_list.sort(ip_sort)
start_idx = 0
end_idx = start_idx+1
cur_interval = [ip_list[start_idx]]
root = ip_list[start_idx]._splitted_start[0]
root_ips = [None] * 256
while True:
if end_idx >= len(ip_list): break
if ip_list[end_idx]._splitted_start[0] != root:
start_idx = end_idx
res = manage_root(root, cur_interval, 1)
print_ip(res)
root_ips[res._splitted_start[0]] = res
cur_interval = [ip_list[end_idx]]
root = ip_list[start_idx]._splitted_start[0]
else:
cur_interval.append(ip_list[end_idx])
end_idx += 1
res = manage_root(root, cur_interval, 1)
print_ip(res)
print '\nstatic const ip_level* %s[256] = {' % (array_name)
for i in range(0, 256):
if root_ips[i]:
print '\t&%s,' % (root_ips[i].name())
else:
print '\tNULL, // %d' % (i)
print '};\n'
build_array(array_vals_ipv4.values(), 's_root_ipv4')
build_array(array_vals_ipv6.values(), 's_root_ipv6')
print 'static const uint8_t country_codes[][3] = {'
for cc in countries:
src/ip_to_geo.c
7070
7171
7272
73
73
74
75
7476
7577
7678
const ip_level* first_level;
if (ip_size == 4)
first_level = s_root_ip[ip[0]];
first_level = s_root_ipv4[ip[0]];
else if (ip_size == 16)
first_level = s_root_ipv6[ip[0]];
else
return NULL;

Archive Download the corresponding diff file

Branches

Tags