腾讯面试：Kafka如何处理百万级消息队列？

Properties props = new Properties();
props.put("bootstrap.servers", "kafka-broker1:9092,kafka-broker2:9092");
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");

Producer<String, String> producer = new KafkaProducer<>(props);
// 发送消息
for(int i = 0; i < 1000000; i++) {
    producer.send(new ProducerRecord<String, String>("my-topic", Integer.toString(i), "message-" + i));
    // my-topic：目标主题
    // Integer.toString(i)：消息的键（key），这里用作分区依据
    // "message-" + i：消息的值（value）
}
producer.close();

Properties props = new Properties();
props.put("bootstrap.servers", "kafka-broker1:9092,kafka-broker2:9092");
props.put("group.id", "my-consumer-group");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
consumer.subscribe(Arrays.asList("my-topic"));
// 订阅主题
while (true) {
    ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
    for (ConsumerRecord<String, String> record : records) {
        System.out.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());
        // 处理消息
    }
}

StreamsBuilder builder = new StreamsBuilder();
KStream<String, String> textLines = builder.stream("my-input-topic");
KTable<String, Long> wordCounts = textLines
    .flatMapValues(textLine -> Arrays.asList(textLine.toLowerCase().split("\\W+")))
    .groupBy((key, word) -> word)
    .count(Materialized.as("counts-store"));
wordCounts.toStream().to("my-output-topic", Produced.with(Serdes.String(), Serdes.Long()));

KafkaStreams streams = new KafkaStreams(builder.build(), props);
streams.start();

// 生产者配置优化
props.put("linger.ms", 10);
props.put("batch.size", 16384);
props.put("buffer.memory", 33554432);

// 消费者配置优化
props.put("fetch.min.bytes", 1024);
props.put("fetch.max.wait.ms", 100);

props.put("compression.type", "snappy");

// 以连接到MySQL数据库为例
// 实际上需要配置Connect的配置文件
{
  "name": "my-connector",
  "config": {
    "connector.class": "io.confluent.connect.jdbc.JdbcSinkConnector",
    "tasks.max": "1",
    "topics": "my-topic",
    "connection.url": "jdbc:mysql://localhost:3306/mydb",
    "key.converter": "org.apache.kafka.connect.json.JsonConverter",
    "value.converter": "org.apache.kafka.connect.json.JsonConverter",
  }
}

// 使用JMX监控Kafka性能指标的示例代码
//具体实现需要根据监控工具的API进行

// 在Kafka配置文件中设置副本因子
broker.id=0
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=2
transaction.state.log.replication.factor=2
transaction.state.log.min.isr=2
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
zookeeper.connection.timeout.ms=6000

props.put("transactional.id", "my-transactional-id");
Producer<String, String> producer = new KafkaProducer<>(props);
producer.initTransactions();

try {
    producer.beginTransaction();
    for(int i = 0; i < 100; i++) {
        producer.send(new ProducerRecord<>("my-topic", Integer.toString(i), "value-" + i));
    }
    producer.commitTransaction();
} catch (ProducerFencedException | OutOfOrderSequenceException | AuthorizationException e) {
    producer.abortTransaction();
} catch (KafkaException e) {
    // 处理异常
}