import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private void refreshShards() {
  DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
  describeStreamRequest.setStreamName(streamName);
  String exclusiveStartShardId = null;
  List<Shard> shards = new ArrayList<>();

  do {
    describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
    DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
    shards.addAll(describeStreamResult.getStreamDescription().getShards());

    if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) {
      exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
    } else {
      exclusiveStartShardId = null;
    }
  } while (exclusiveStartShardId != null);

  this.shards = shards;
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
@Ignore("Kinesalite doesn't support updateShardCount. Test only against real AWS Kinesis")
public void testPartitionCountIncreasedIfAutoAddPartitionsSet() {
	KinesisBinderConfigurationProperties configurationProperties = new KinesisBinderConfigurationProperties();

	String stream = "existing" + System.currentTimeMillis();

	AmazonKinesisAsync amazonKinesis = localKinesisResource.getResource();
	amazonKinesis.createStream(stream, 1);

	List<Shard> shards = describeStream(stream);

	assertThat(shards.size()).isEqualTo(1);

	configurationProperties.setMinShardCount(6);
	configurationProperties.setAutoAddShards(true);
	KinesisTestBinder binder = getBinder(configurationProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
	Binding<?> binding = binder.bindConsumer(stream, "test", new NullChannel(), consumerProperties);
	binding.unbind();

	shards = describeStream(stream);

	assertThat(shards.size()).isEqualTo(6);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void testProvisionProducerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(new KinesisProducerProperties());
	String name = "test-stream";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ProducerDestination destination = provisioner.provisionProducerDestination(name, extendedProducerProperties);

	verify(amazonKinesisMock)
			.describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void testProvisionConsumerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ConsumerDestination destination =
			provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock)
			.describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
public List<Shard> listShards(final String streamName) throws TransientKinesisException {
  return wrapExceptions(new Callable<List<Shard>>() {

    @Override
    public List<Shard> call() throws Exception {
      List<Shard> shards = Lists.newArrayList();
      String lastShardId = null;

      StreamDescription description;
      do {
        description = kinesis.describeStream(streamName, lastShardId)
            .getStreamDescription();

        shards.addAll(description.getShards());
        lastShardId = shards.get(shards.size() - 1).getShardId();
      } while (description.getHasMoreShards());

      return shards;
    }
  });
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void shouldListAllShards() throws Exception {
  Shard shard1 = new Shard().withShardId(SHARD_1);
  Shard shard2 = new Shard().withShardId(SHARD_2);
  Shard shard3 = new Shard().withShardId(SHARD_3);
  given(kinesis.describeStream(STREAM, null)).willReturn(new DescribeStreamResult()
      .withStreamDescription(new StreamDescription()
          .withShards(shard1, shard2)
          .withHasMoreShards(true)));
  given(kinesis.describeStream(STREAM, SHARD_2)).willReturn(new DescribeStreamResult()
      .withStreamDescription(new StreamDescription()
          .withShards(shard3)
          .withHasMoreShards(false)));

  List<Shard> shards = underTest.listShards(STREAM);

  assertThat(shards).containsOnly(shard1, shard2, shard3);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Override
public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId) {
  int nextShardId = 0;
  if (exclusiveStartShardId != null) {
    nextShardId = parseInt(exclusiveStartShardId) + 1;
  }
  boolean hasMoreShards = nextShardId + 1 < shardedData.size();

  List<Shard> shards = new ArrayList<>();
  if (nextShardId < shardedData.size()) {
    shards.add(new Shard().withShardId(Integer.toString(nextShardId)));
  }

  HttpResponse response = new HttpResponse(null, null);
  response.setStatusCode(200);
  DescribeStreamResult result = new DescribeStreamResult();
  result.setSdkHttpMetadata(SdkHttpMetadata.from(response));
  result.withStreamDescription(
      new StreamDescription()
          .withHasMoreShards(hasMoreShards)
          .withShards(shards)
          .withStreamName(streamName));
  return result;
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
/**
 * Utility function to convert {@link StreamShardMetadata} into {@link StreamShardHandle}.
 *
 * @param streamShardMetadata the {@link StreamShardMetadata} to be converted
 * @return a {@link StreamShardHandle} object
 */
public static StreamShardHandle convertToStreamShardHandle(StreamShardMetadata streamShardMetadata) {
	Shard shard = new Shard();
	shard.withShardId(streamShardMetadata.getShardId());
	shard.withParentShardId(streamShardMetadata.getParentShardId());
	shard.withAdjacentParentShardId(streamShardMetadata.getAdjacentParentShardId());

	HashKeyRange hashKeyRange = new HashKeyRange();
	hashKeyRange.withStartingHashKey(streamShardMetadata.getStartingHashKey());
	hashKeyRange.withEndingHashKey(streamShardMetadata.getEndingHashKey());
	shard.withHashKeyRange(hashKeyRange);

	SequenceNumberRange sequenceNumberRange = new SequenceNumberRange();
	sequenceNumberRange.withStartingSequenceNumber(streamShardMetadata.getStartingSequenceNumber());
	sequenceNumberRange.withEndingSequenceNumber(streamShardMetadata.getEndingSequenceNumber());
	shard.withSequenceNumberRange(sequenceNumberRange);

	return new StreamShardHandle(streamShardMetadata.getStreamName(), shard);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private List<StreamShardHandle> getShardsOfStream(String streamName, @Nullable String lastSeenShardId) throws InterruptedException {
	List<StreamShardHandle> shardsOfStream = new ArrayList<>();

	DescribeStreamResult describeStreamResult;
	do {
		describeStreamResult = describeStream(streamName, lastSeenShardId);

		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		for (Shard shard : shards) {
			shardsOfStream.add(new StreamShardHandle(streamName, shard));
		}

		if (shards.size() != 0) {
			lastSeenShardId = shards.get(shards.size() - 1).getShardId();
		}
	} while (describeStreamResult.getStreamDescription().isHasMoreShards());

	return shardsOfStream;
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
public NonReshardedStreamsKinesis(Map<String, Integer> streamsToShardCount) {
	for (Map.Entry<String, Integer> streamToShardCount : streamsToShardCount.entrySet()) {
		String streamName = streamToShardCount.getKey();
		int shardCount = streamToShardCount.getValue();

		if (shardCount == 0) {
			// don't do anything
		} else {
			List<StreamShardHandle> shardsOfStream = new ArrayList<>(shardCount);
			for (int i = 0; i < shardCount; i++) {
				shardsOfStream.add(
					new StreamShardHandle(
						streamName,
						new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(i))));
			}
			streamsWithListOfShards.put(streamName, shardsOfStream);
		}
	}
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private List<KinesisStreamShard> getShardsOfStream(String streamName, @Nullable String lastSeenShardId) throws InterruptedException {
	List<KinesisStreamShard> shardsOfStream = new ArrayList<>();

	DescribeStreamResult describeStreamResult;
	do {
		describeStreamResult = describeStream(streamName, lastSeenShardId);

		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		for (Shard shard : shards) {
			shardsOfStream.add(new KinesisStreamShard(streamName, shard));
		}

		if (shards.size() != 0) {
			lastSeenShardId = shards.get(shards.size() - 1).getShardId();
		}
	} while (describeStreamResult.getStreamDescription().isHasMoreShards());

	return shardsOfStream;
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
public NonReshardedStreamsKinesis(Map<String,Integer> streamsToShardCount) {
	for (Map.Entry<String,Integer> streamToShardCount : streamsToShardCount.entrySet()) {
		String streamName = streamToShardCount.getKey();
		int shardCount = streamToShardCount.getValue();

		if (shardCount == 0) {
			// don't do anything
		} else {
			List<KinesisStreamShard> shardsOfStream = new ArrayList<>(shardCount);
			for (int i=0; i < shardCount; i++) {
				shardsOfStream.add(
					new KinesisStreamShard(
						streamName,
						new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(i))));
			}
			streamsWithListOfShards.put(streamName, shardsOfStream);
		}
	}
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Before
public void setup() throws Exception {
    KinesisEndpoint endpoint = new KinesisEndpoint(null, "streamName", component);
    endpoint.setAmazonKinesisClient(kinesisClient);
    endpoint.setIteratorType(ShardIteratorType.LATEST);
    undertest = new KinesisConsumer(endpoint, processor);

    when(kinesisClient.getRecords(any(GetRecordsRequest.class)))
        .thenReturn(new GetRecordsResult()
            .withNextShardIterator("nextShardIterator")
        );
    when(kinesisClient.describeStream(any(DescribeStreamRequest.class)))
        .thenReturn(new DescribeStreamResult()
            .withStreamDescription(new StreamDescription()
                .withShards(new Shard().withShardId("shardId"))
            )
        );
    when(kinesisClient.getShardIterator(any(GetShardIteratorRequest.class)))
        .thenReturn(new GetShardIteratorResult()
            .withShardIterator("shardIterator")
        );
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout)
{
    KinesisTableLayoutHandle kinesislayout = handleResolver.convertLayout(layout);
    KinesisTableHandle kinesisTableHandle = kinesislayout.getTable();

    InternalStreamDescription desc = this.getStreamDescription(kinesisTableHandle.getStreamName());

    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    for (Shard shard : desc.getShards()) {
        KinesisSplit split = new KinesisSplit(connectorId,
                kinesisTableHandle.getStreamName(),
                kinesisTableHandle.getMessageDataFormat(),
                shard.getShardId(),
                shard.getSequenceNumberRange().getStartingSequenceNumber(),
                shard.getSequenceNumberRange().getEndingSequenceNumber());
        builder.add(split);
    }

    return new FixedSplitSource(builder.build());
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private List<Shard> describeStream(String stream) {
	AmazonKinesisAsync amazonKinesis = localKinesisResource.getResource();

	String exclusiveStartShardId = null;

	DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
			.withStreamName(stream);

	List<Shard> shardList = new ArrayList<>();

	while (true) {
		DescribeStreamResult describeStreamResult = null;

		describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
		describeStreamResult = amazonKinesis.describeStream(describeStreamRequest);
		StreamDescription streamDescription = describeStreamResult.getStreamDescription();
		if (StreamStatus.ACTIVE.toString().equals(streamDescription.getStreamStatus())) {
			shardList.addAll(streamDescription.getShards());

			if (streamDescription.getHasMoreShards()) {
				exclusiveStartShardId = shardList.get(shardList.size() - 1).getShardId();
				continue;
			}
			else {
				return shardList;
			}
		}
		try {
			Thread.sleep(100);
		}
		catch (InterruptedException e) {
			Thread.currentThread().interrupt();
			throw new IllegalStateException(e);
		}
	}
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void testProvisionConsumerExistingStreamUpdateShards() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	ArgumentCaptor<UpdateShardCountRequest> updateShardCaptor =
			ArgumentCaptor.forClass(UpdateShardCountRequest.class);
	String name = "test-stream";
	String group = "test-group";
	int targetShardCount = 2;
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setMinShardCount(targetShardCount);
	binderProperties.setAutoAddShards(true);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());

	DescribeStreamResult describeOriginalStream =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	DescribeStreamResult describeUpdatedStream =
			describeStreamResultWithShards(Arrays.asList(new Shard(), new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeOriginalStream)
			.thenReturn(describeUpdatedStream);

	provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(1))
			.updateShardCount(updateShardCaptor.capture());

	assertThat(updateShardCaptor.getValue().getStreamName()).isEqualTo(name);
	assertThat(updateShardCaptor.getValue().getScalingType()).isEqualTo(ScalingType.UNIFORM_SCALING.name());
	assertThat(updateShardCaptor.getValue().getTargetShardCount()).isEqualTo(targetShardCount);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void testProvisionProducerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(new KinesisProducerProperties());

	String name = "test-stream";
	Integer shards = 1;

	DescribeStreamResult describeStreamResult =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, shards))
			.thenReturn(new CreateStreamResult());

	ProducerDestination destination = provisioner.provisionProducerDestination(name, extendedProducerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock)
			.createStream(name, shards);

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void testProvisionProducerUpdateShards() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	ArgumentCaptor<UpdateShardCountRequest> updateShardCaptor = ArgumentCaptor.forClass(UpdateShardCountRequest.class);
	String name = "test-stream";
	String group = "test-group";
	int targetShardCount = 2;
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setMinShardCount(targetShardCount);
	binderProperties.setAutoAddShards(true);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());

	DescribeStreamResult describeOriginalStream =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));

	DescribeStreamResult describeUpdatedStream =
			describeStreamResultWithShards(Arrays.asList(new Shard(), new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeOriginalStream)
			.thenReturn(describeUpdatedStream);

	provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(1))
			.updateShardCount(updateShardCaptor.capture());
	assertThat(updateShardCaptor.getValue().getStreamName()).isEqualTo(name);
	assertThat(updateShardCaptor.getValue().getScalingType()).isEqualTo(ScalingType.UNIFORM_SCALING.name());
	assertThat(updateShardCaptor.getValue().getTargetShardCount()).isEqualTo(targetShardCount);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
@Test
public void testProvisionConsumerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult =
			describeStreamResultWithShards(Collections.singletonList(new Shard()));


	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, instanceCount * concurrency))
			.thenReturn(new CreateStreamResult());

	ConsumerDestination destination = provisioner.provisionConsumerDestination(name, group,
			extendedConsumerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock)
			.createStream(name, instanceCount * concurrency);

	assertThat(destination.getName()).isEqualTo(name);
}
 

import com.amazonaws.services.kinesis.model.Shard; //导入依赖的package包/类
private static DescribeStreamResult describeStreamResultWithShards(List<Shard> shards) {
	return new DescribeStreamResult()
			.withStreamDescription(
					new StreamDescription()
							.withShards(shards)
							.withStreamStatus(StreamStatus.ACTIVE)
							.withHasMoreShards(Boolean.FALSE));
}