Home Explore Help
Register Sign In
heavensend
/
myproject
Watch 0
Star 0
Fork 0
Files Issues 1 Pull Requests 0 Wiki
Tree: 71b840affd
Branches Tags
myproject
/
ansible
/
roles
/
flume
/
files
/
conf
/
nginx.access.conf

nginx.access.conf 22 KB
History Raw

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322 
  1. agent01.sources = source1 source2 source3 source4 source5 source6 source7 source8
    2. 

  2. agent01.channels = sink01 sink02
    3. 

  3. agent01.sinks = elasticSearch
    4. 

  4. 

  5. #agent01.sources.source1.type = exec 
    6. 

  6. #agent01.sources.source1.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    7. 

  7. #agent01.sources.source1.restart = true
    8. 

  8. #agent01.sources.source1.logStdErr = true
    9. 

  9. #agent01.sources.source1.batchSize = 500
    10. 

  10. #agent01.sources.source1.channels = sink02
    11. 

  11. 

  12. 

  13. ####source1######
    14. 

  14. agent01.sources.source1.type = exec
    15. 

  15. agent01.sources.source1.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    16. 

  16. agent01.sources.source1.restart = true
    17. 

  17. agent01.sources.source1.logStdErr = true
    18. 

  18. agent01.sources.source1.channels = sink02
    19. 

  19. 

  20. agent01.sources.source1.interceptors = interceptor1 interceptor2 interceptor3
    21. 

  21. agent01.sources.source1.interceptors.interceptor1.type = host
    22. 

  22. agent01.sources.source1.interceptors.interceptor1.hostHeader = host
    23. 

  23. 

  24. agent01.sources.source1.interceptors.interceptor2.type = regex_extractor
    25. 

  25. agent01.sources.source1.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    26. 

  26. agent01.sources.source1.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    27. 

  27. agent01.sources.source1.interceptors.interceptor2.serializers.s1.name = remote_addr
    28. 

  28. agent01.sources.source1.interceptors.interceptor2.serializers.s2.name = remote_user
    29. 

  29. agent01.sources.source1.interceptors.interceptor2.serializers.s3.name = datetime
    30. 

  30. #这里的时间已经是ISO8601格式，kibana可以直接识别为时间格式，所以下面的3行可以不用
    31. 

  31. #agent01.sources.source1.interceptors.interceptor2.serializers.s3.type = org.apache.flume.interceptor.RegexExtractorInterceptorMillisSerializer
    32. 

  32. #agent01.sources.source1.interceptors.interceptor2.serializers.s3.name = timestamp
    33. 

  33. #agent01.sources.source1.interceptors.interceptor2.serializers.s3.pattern  = yyyy-MM-dd'T'HH:mm:ssZ
    34. 

  34. agent01.sources.source1.interceptors.interceptor2.serializers.s4.name = http_method
    35. 

  35. agent01.sources.source1.interceptors.interceptor2.serializers.s5.name = uri
    36. 

  36. agent01.sources.source1.interceptors.interceptor2.serializers.s6.name = status
    37. 

  37. agent01.sources.source1.interceptors.interceptor2.serializers.s7.name = body_length
    38. 

  38. agent01.sources.source1.interceptors.interceptor2.serializers.s8.name = http_referer
    39. 

  39. agent01.sources.source1.interceptors.interceptor2.serializers.s9.name = user_agent
    40. 

  40. agent01.sources.source1.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    41. 

  41. agent01.sources.source1.interceptors.interceptor2.serializers.s11.name = request_time
    42. 

  42. agent01.sources.source1.interceptors.interceptor2.serializers.s12.name = upstream_addr
    43. 

  43. agent01.sources.source1.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    44. 

  44. agent01.sources.source1.interceptors.interceptor2.serializers.s14.name = post_body
    45. 

  45. agent01.sources.source1.interceptors.interceptor2.serializers.s15.name = domain_url
    46. 

  46. 

  47. agent01.sources.source1.interceptors.interceptor3.type = timestamp
    48. 

  48. 

  49. 

  50. ####source2######
    51. 

  51. agent01.sources.source2.type = exec
    52. 

  52. agent01.sources.source2.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    53. 

  53. agent01.sources.source2.channels = sink02
    54. 

  54. 

  55. agent01.sources.source2.interceptors = interceptor1 interceptor2 interceptor3
    56. 

  56. agent01.sources.source2.interceptors.interceptor1.type = host
    57. 

  57. agent01.sources.source2.interceptors.interceptor1.hostHeader = host
    58. 

  58. 

  59. agent01.sources.source2.interceptors.interceptor2.type = regex_extractor
    60. 

  60. agent01.sources.source2.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    61. 

  61. agent01.sources.source2.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    62. 

  62. agent01.sources.source2.interceptors.interceptor2.serializers.s1.name = remote_addr
    63. 

  63. agent01.sources.source2.interceptors.interceptor2.serializers.s2.name = remote_user
    64. 

  64. agent01.sources.source2.interceptors.interceptor2.serializers.s3.name = datetime
    65. 

  65. agent01.sources.source2.interceptors.interceptor2.serializers.s4.name = http_method
    66. 

  66. agent01.sources.source2.interceptors.interceptor2.serializers.s5.name = uri
    67. 

  67. agent01.sources.source2.interceptors.interceptor2.serializers.s6.name = status
    68. 

  68. agent01.sources.source2.interceptors.interceptor2.serializers.s7.name = body_length
    69. 

  69. agent01.sources.source2.interceptors.interceptor2.serializers.s8.name = http_referer
    70. 

  70. agent01.sources.source2.interceptors.interceptor2.serializers.s9.name = user_agent
    71. 

  71. agent01.sources.source2.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    72. 

  72. agent01.sources.source2.interceptors.interceptor2.serializers.s11.name = request_time
    73. 

  73. agent01.sources.source2.interceptors.interceptor2.serializers.s12.name = upstream_addr
    74. 

  74. agent01.sources.source2.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    75. 

  75. agent01.sources.source2.interceptors.interceptor2.serializers.s14.name = post_body
    76. 

  76. agent01.sources.source2.interceptors.interceptor2.serializers.s15.name = domain_url
    77. 

  77. 

  78. agent01.sources.source2.interceptors.interceptor3.type = timestamp
    79. 

  79. 

  80. ####source3######
    81. 

  81. agent01.sources.source3.type = exec
    82. 

  82. agent01.sources.source3.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    83. 

  83. agent01.sources.source3.channels = sink02
    84. 

  84. 

  85. agent01.sources.source3.interceptors = interceptor1 interceptor2 interceptor3
    86. 

  86. agent01.sources.source3.interceptors.interceptor1.type = host
    87. 

  87. agent01.sources.source3.interceptors.interceptor1.hostHeader = host
    88. 

  88. 

  89. agent01.sources.source3.interceptors.interceptor2.type = regex_extractor
    90. 

  90. agent01.sources.source3.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    91. 

  91. agent01.sources.source3.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    92. 

  92. agent01.sources.source3.interceptors.interceptor2.serializers.s1.name = remote_addr
    93. 

  93. agent01.sources.source3.interceptors.interceptor2.serializers.s2.name = remote_user
    94. 

  94. agent01.sources.source3.interceptors.interceptor2.serializers.s3.name = datetime
    95. 

  95. agent01.sources.source3.interceptors.interceptor2.serializers.s4.name = http_method
    96. 

  96. agent01.sources.source3.interceptors.interceptor2.serializers.s5.name = uri
    97. 

  97. agent01.sources.source3.interceptors.interceptor2.serializers.s6.name = status
    98. 

  98. agent01.sources.source3.interceptors.interceptor2.serializers.s7.name = body_length
    99. 

  99. agent01.sources.source3.interceptors.interceptor2.serializers.s8.name = http_referer
    100. 

  100. agent01.sources.source3.interceptors.interceptor2.serializers.s9.name = user_agent
    101. 

  101. agent01.sources.source3.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    102. 

  102. agent01.sources.source3.interceptors.interceptor2.serializers.s11.name = request_time
    103. 

  103. agent01.sources.source3.interceptors.interceptor2.serializers.s12.name = upstream_addr
    104. 

  104. agent01.sources.source3.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    105. 

  105. agent01.sources.source3.interceptors.interceptor2.serializers.s14.name = post_body
    106. 

  106. agent01.sources.source3.interceptors.interceptor2.serializers.s15.name = domain_url
    107. 

  107. 

  108. agent01.sources.source3.interceptors.interceptor3.type = timestamp
    109. 

  109. 

  110. 

  111. ####source4######
    112. 

  112. agent01.sources.source4.type = exec
    113. 

  113. agent01.sources.source4.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    114. 

  114. agent01.sources.source4.channels = sink02
    115. 

  115. 

  116. agent01.sources.source4.interceptors = interceptor1 interceptor2 interceptor3
    117. 

  117. agent01.sources.source4.interceptors.interceptor1.type = host
    118. 

  118. agent01.sources.source4.interceptors.interceptor1.hostHeader = host
    119. 

  119. 

  120. agent01.sources.source4.interceptors.interceptor2.type = regex_extractor
    121. 

  121. agent01.sources.source4.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    122. 

  122. agent01.sources.source4.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    123. 

  123. agent01.sources.source4.interceptors.interceptor2.serializers.s1.name = remote_addr
    124. 

  124. agent01.sources.source4.interceptors.interceptor2.serializers.s2.name = remote_user
    125. 

  125. agent01.sources.source4.interceptors.interceptor2.serializers.s3.name = datetime
    126. 

  126. agent01.sources.source4.interceptors.interceptor2.serializers.s4.name = http_method
    127. 

  127. agent01.sources.source4.interceptors.interceptor2.serializers.s5.name = uri
    128. 

  128. agent01.sources.source4.interceptors.interceptor2.serializers.s6.name = status
    129. 

  129. agent01.sources.source4.interceptors.interceptor2.serializers.s7.name = body_length
    130. 

  130. agent01.sources.source4.interceptors.interceptor2.serializers.s8.name = http_referer
    131. 

  131. agent01.sources.source4.interceptors.interceptor2.serializers.s9.name = user_agent
    132. 

  132. agent01.sources.source4.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    133. 

  133. agent01.sources.source4.interceptors.interceptor2.serializers.s11.name = request_time
    134. 

  134. agent01.sources.source4.interceptors.interceptor2.serializers.s12.name = upstream_addr
    135. 

  135. agent01.sources.source4.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    136. 

  136. agent01.sources.source4.interceptors.interceptor2.serializers.s14.name = post_body
    137. 

  137. agent01.sources.source4.interceptors.interceptor2.serializers.s15.name = domain_url
    138. 

  138. 

  139. agent01.sources.source4.interceptors.interceptor3.type = timestamp
    140. 

  140. 

  141. ###source5######
    142. 

  142. agent01.sources.source5.type = exec
    143. 

  143. agent01.sources.source5.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    144. 

  144. agent01.sources.source5.channels = sink02
    145. 

  145. 

  146. agent01.sources.source5.interceptors = interceptor1 interceptor2 interceptor3
    147. 

  147. agent01.sources.source5.interceptors.interceptor1.type = host
    148. 

  148. agent01.sources.source5.interceptors.interceptor1.hostHeader = host
    149. 

  149. 

  150. agent01.sources.source5.interceptors.interceptor2.type = regex_extractor
    151. 

  151. agent01.sources.source5.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    152. 

  152. agent01.sources.source5.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    153. 

  153. agent01.sources.source5.interceptors.interceptor2.serializers.s1.name = remote_addr
    154. 

  154. agent01.sources.source5.interceptors.interceptor2.serializers.s2.name = remote_user
    155. 

  155. agent01.sources.source5.interceptors.interceptor2.serializers.s3.name = datetime
    156. 

  156. agent01.sources.source5.interceptors.interceptor2.serializers.s4.name = http_method
    157. 

  157. agent01.sources.source5.interceptors.interceptor2.serializers.s5.name = uri
    158. 

  158. agent01.sources.source5.interceptors.interceptor2.serializers.s6.name = status
    159. 

  159. agent01.sources.source5.interceptors.interceptor2.serializers.s7.name = body_length
    160. 

  160. agent01.sources.source5.interceptors.interceptor2.serializers.s8.name = http_referer
    161. 

  161. agent01.sources.source5.interceptors.interceptor2.serializers.s9.name = user_agent
    162. 

  162. agent01.sources.source5.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    163. 

  163. agent01.sources.source5.interceptors.interceptor2.serializers.s11.name = request_time
    164. 

  164. agent01.sources.source5.interceptors.interceptor2.serializers.s12.name = upstream_addr
    165. 

  165. agent01.sources.source5.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    166. 

  166. agent01.sources.source5.interceptors.interceptor2.serializers.s14.name = post_body
    167. 

  167. agent01.sources.source5.interceptors.interceptor2.serializers.s15.name = domain_url
    168. 

  168. 

  169. agent01.sources.source5.interceptors.interceptor3.type = timestamp
    170. 

  170. 

  171. 

  172. ##source6######
    173. 

  173. agent01.sources.source6.type = exec
    174. 

  174. agent01.sources.source6.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    175. 

  175. agent01.sources.source6.channels = sink02
    176. 

  176. 

  177. agent01.sources.source6.interceptors = interceptor1 interceptor2 interceptor3
    178. 

  178. agent01.sources.source6.interceptors.interceptor1.type = host
    179. 

  179. agent01.sources.source6.interceptors.interceptor1.hostHeader = host
    180. 

  180. 

  181. agent01.sources.source6.interceptors.interceptor2.type = regex_extractor
    182. 

  182. agent01.sources.source6.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    183. 

  183. agent01.sources.source6.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    184. 

  184. agent01.sources.source6.interceptors.interceptor2.serializers.s1.name = remote_addr
    185. 

  185. agent01.sources.source6.interceptors.interceptor2.serializers.s2.name = remote_user
    186. 

  186. agent01.sources.source6.interceptors.interceptor2.serializers.s3.name = datetime
    187. 

  187. agent01.sources.source6.interceptors.interceptor2.serializers.s4.name = http_method
    188. 

  188. agent01.sources.source6.interceptors.interceptor2.serializers.s5.name = uri
    189. 

  189. agent01.sources.source6.interceptors.interceptor2.serializers.s6.name = status
    190. 

  190. agent01.sources.source6.interceptors.interceptor2.serializers.s7.name = body_length
    191. 

  191. agent01.sources.source6.interceptors.interceptor2.serializers.s8.name = http_referer
    192. 

  192. agent01.sources.source6.interceptors.interceptor2.serializers.s9.name = user_agent
    193. 

  193. agent01.sources.source6.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    194. 

  194. agent01.sources.source6.interceptors.interceptor2.serializers.s11.name = request_time
    195. 

  195. agent01.sources.source6.interceptors.interceptor2.serializers.s12.name = upstream_addr
    196. 

  196. agent01.sources.source6.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    197. 

  197. agent01.sources.source6.interceptors.interceptor2.serializers.s14.name = post_body
    198. 

  198. agent01.sources.source6.interceptors.interceptor2.serializers.s15.name = domain_url
    199. 

  199. 

  200. agent01.sources.source6.interceptors.interceptor3.type = timestamp
    201. 

  201. 

  202. 

  203. ######source7######
    204. 

  204. agent01.sources.source7.type = exec
    205. 

  205. agent01.sources.source7.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    206. 

  206. agent01.sources.source7.channels = sink02
    207. 

  207. 

  208. agent01.sources.source7.interceptors = interceptor1 interceptor2 interceptor3
    209. 

  209. agent01.sources.source7.interceptors.interceptor1.type = host
    210. 

  210. agent01.sources.source7.interceptors.interceptor1.hostHeader = host
    211. 

  211. 

  212. agent01.sources.source7.interceptors.interceptor2.type = regex_extractor
    213. 

  213. agent01.sources.source7.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    214. 

  214. agent01.sources.source7.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    215. 

  215. agent01.sources.source7.interceptors.interceptor2.serializers.s1.name = remote_addr
    216. 

  216. agent01.sources.source7.interceptors.interceptor2.serializers.s2.name = remote_user
    217. 

  217. agent01.sources.source7.interceptors.interceptor2.serializers.s3.name = datetime
    218. 

  218. agent01.sources.source7.interceptors.interceptor2.serializers.s4.name = http_method
    219. 

  219. agent01.sources.source7.interceptors.interceptor2.serializers.s5.name = uri
    220. 

  220. agent01.sources.source7.interceptors.interceptor2.serializers.s6.name = status
    221. 

  221. agent01.sources.source7.interceptors.interceptor2.serializers.s7.name = body_length
    222. 

  222. agent01.sources.source7.interceptors.interceptor2.serializers.s8.name = http_referer
    223. 

  223. agent01.sources.source7.interceptors.interceptor2.serializers.s9.name = user_agent
    224. 

  224. agent01.sources.source7.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    225. 

  225. agent01.sources.source7.interceptors.interceptor2.serializers.s11.name = request_time
    226. 

  226. agent01.sources.source7.interceptors.interceptor2.serializers.s12.name = upstream_addr
    227. 

  227. agent01.sources.source7.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    228. 

  228. agent01.sources.source7.interceptors.interceptor2.serializers.s14.name = post_body
    229. 

  229. agent01.sources.source7.interceptors.interceptor2.serializers.s15.name = domain_url
    230. 

  230. 

  231. agent01.sources.source7.interceptors.interceptor3.type = timestamp
    232. 

  232. 

  233. 

  234. #####source8######
    235. 

  235. agent01.sources.source8.type = exec
    236. 

  236. agent01.sources.source8.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    237. 

  237. agent01.sources.source8.channels = sink02
    238. 

  238. 

  239. agent01.sources.source8.interceptors = interceptor1 interceptor2 interceptor3
    240. 

  240. agent01.sources.source8.interceptors.interceptor1.type = host
    241. 

  241. agent01.sources.source8.interceptors.interceptor1.hostHeader = host
    242. 

  242. 

  243. agent01.sources.source8.interceptors.interceptor2.type = regex_extractor
    244. 

  244. agent01.sources.source8.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    245. 

  245. agent01.sources.source8.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    246. 

  246. agent01.sources.source8.interceptors.interceptor2.serializers.s1.name = remote_addr
    247. 

  247. agent01.sources.source8.interceptors.interceptor2.serializers.s2.name = remote_user
    248. 

  248. agent01.sources.source8.interceptors.interceptor2.serializers.s3.name = datetime
    249. 

  249. agent01.sources.source8.interceptors.interceptor2.serializers.s4.name = http_method
    250. 

  250. agent01.sources.source8.interceptors.interceptor2.serializers.s5.name = uri
    251. 

  251. agent01.sources.source8.interceptors.interceptor2.serializers.s6.name = status
    252. 

  252. agent01.sources.source8.interceptors.interceptor2.serializers.s7.name = body_length
    253. 

  253. agent01.sources.source8.interceptors.interceptor2.serializers.s8.name = http_referer
    254. 

  254. agent01.sources.source8.interceptors.interceptor2.serializers.s9.name = user_agent
    255. 

  255. agent01.sources.source8.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    256. 

  256. agent01.sources.source8.interceptors.interceptor2.serializers.s11.name = request_time
    257. 

  257. agent01.sources.source8.interceptors.interceptor2.serializers.s12.name = upstream_addr
    258. 

  258. agent01.sources.source8.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    259. 

  259. agent01.sources.source8.interceptors.interceptor2.serializers.s14.name = post_body
    260. 

  260. agent01.sources.source8.interceptors.interceptor2.serializers.s15.name = domain_url
    261. 

  261. 

  262. agent01.sources.source8.interceptors.interceptor3.type = timestamp
    263. 

  263. 

  264. #####source9######
    265. 

  265. agent01.sources.source9.type = exec
    266. 

  266. agent01.sources.source9.command = tail -F /usr/local/openresty/nginx/logs/some_access.log
    267. 

  267. agent01.sources.source9.channels = sink02
    268. 

  268. 

  269. agent01.sources.source9.interceptors = interceptor1 interceptor2 interceptor3
    270. 

  270. agent01.sources.source9.interceptors.interceptor1.type = host
    271. 

  271. agent01.sources.source9.interceptors.interceptor1.hostHeader = host
    272. 

  272. 

  273. agent01.sources.source9.interceptors.interceptor2.type = regex_extractor
    274. 

  274. agent01.sources.source9.interceptors.interceptor2.regex = ([^\\s]*)\\s-\\s([^\\s]*)\\s\\[(.*)\\]\\s+\\"([\\S]*)\\s+([\\S]*)\\s+[\\S]*\\"\\s+(\\d+)\\s+(\\d+)\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"([^\\"]*)\\"\\s+\\"(.*?)\\"\\s+\\"([^\\"]*)\\"
    275. 

  275. agent01.sources.source9.interceptors.interceptor2.serializers = s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11 s12 s13 s14 s15
    276. 

  276. agent01.sources.source9.interceptors.interceptor2.serializers.s1.name = remote_addr
    277. 

  277. agent01.sources.source9.interceptors.interceptor2.serializers.s2.name = remote_user
    278. 

  278. agent01.sources.source9.interceptors.interceptor2.serializers.s3.name = datetime
    279. 

  279. agent01.sources.source9.interceptors.interceptor2.serializers.s4.name = http_method
    280. 

  280. agent01.sources.source9.interceptors.interceptor2.serializers.s5.name = uri
    281. 

  281. agent01.sources.source9.interceptors.interceptor2.serializers.s6.name = status
    282. 

  282. agent01.sources.source9.interceptors.interceptor2.serializers.s7.name = body_length
    283. 

  283. agent01.sources.source9.interceptors.interceptor2.serializers.s8.name = http_referer
    284. 

  284. agent01.sources.source9.interceptors.interceptor2.serializers.s9.name = user_agent
    285. 

  285. agent01.sources.source9.interceptors.interceptor2.serializers.s10.name = http_x_forwarded_for
    286. 

  286. agent01.sources.source9.interceptors.interceptor2.serializers.s11.name = request_time
    287. 

  287. agent01.sources.source9.interceptors.interceptor2.serializers.s12.name = upstream_addr
    288. 

  288. agent01.sources.source9.interceptors.interceptor2.serializers.s13.name = upstream_response_time
    289. 

  289. agent01.sources.source9.interceptors.interceptor2.serializers.s14.name = post_body
    290. 

  290. agent01.sources.source9.interceptors.interceptor2.serializers.s15.name = domain_url
    291. 

  291. 

  292. agent01.sources.source9.interceptors.interceptor3.type = timestamp
    293. 

  293. 

  294. 

  295. 

  296. agent01.sinks.elasticSearch.type = org.apache.flume.sink.elasticsearch.ElasticSearchSink
    297. 

  297. agent01.sinks.elasticSearch.timeZone=Asia/Shanghai
    298. 

  298. agent01.sinks.elasticSearch.channel = sink02
    299. 

  299. agent01.sinks.elasticSearch.batchSize = 2000
    300. 

  300. agent01.sinks.elasticSearch.hostNames = 10.31.88.120:9300 
    301. 

  301. agent01.sinks.elasticSearch.indexName = nginx_access_log
    302. 

  302. agent01.sinks.elasticSearch.indexType = static
    303. 

  303. agent01.sinks.elasticSearch.clusterName = elasticsearch-zzb1
    304. 

  304. agent01.sinks.elasticSearch.client = transport
    305. 

  305. agent01.sinks.elasticSearch.serializer = org.apache.flume.sink.elasticsearch.ElasticSearchLogStashEventSerializer
    306. 

  306. 

  307. # Each sink's type must be defined
    308. 

  308. agent01.sinks.loggerSink.type = logger
    309. 

  309. 

  310. #Specify the channel the sink should use
    311. 

  311. agent01.sinks.loggerSink.channel = sink01
    312. 

  312. 

  313. # Each channel's type is defined.
    314. 

  314. agent01.channels.sink01.type = memory
    315. 

  315. agent01.channels.sink01.capacity = 10000
    316. 

  316. agent01.channels.sink01.transactionCapacity = 10000
    317. 

  317. agent01.channels.sink01.byteCapacityBufferPercentage = 20
    318. 

  318. agent01.channels.sink01.keep-alive = 30
    319. 

  319. 

  320. agent01.channels.sink02.type = file
    321. 

  321. agent01.channels.sink02.checkpointDir = /data/flume/data/checkpointDir 
    322. 

  322. agent01.channels.sink02.dataDirs = /data/flume/data/dataDirs
    323.