From a9644a70e80ef24041138186a19cb9f92aa12d7e Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Thu, 8 Aug 2019 15:45:05 -0400 Subject: [PATCH 01/72] eth: Port blockwatcher and tests Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 0acd88bbd8c555bebbfc4be9775ba32ed6619cf2 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 581 +++++++++++++++++++++++++++ eth/blockwatch/block_watcher_test.go | 487 ++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 eth/blockwatch/block_watcher.go create mode 100644 eth/blockwatch/block_watcher_test.go diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go new file mode 100644 index 0000000..d2a0a00 --- /dev/null +++ b/eth/blockwatch/block_watcher.go @@ -0,0 +1,581 @@ +package blockwatch + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/rpc" + "github.com/golang/glog" +) + +// maxBlocksInGetLogsQuery is the max number of blocks to fetch logs for in a single query. There is +// a hard limit of 10,000 logs returned by a single `eth_getLogs` query by Infura's Ethereum nodes so +// we need to try and stay below it. Parity, Geth and Alchemy all have much higher limits (if any) on +// the number of logs returned so Infura is by far the limiting factor. +var maxBlocksInGetLogsQuery = 60 + +// EventType describes the types of events emitted by blockwatch.Watcher. A block can be discovered +// and added to our representation of the chain. During a block re-org, a block previously stored +// can be removed from the list. +type EventType int + +const ( + // Added indicates that an event was added to our representation of the chain + Added EventType = iota + // Removed indicates that an event was removed from our representation of the chain + Removed +) + +// Event describes a block event emitted by a Watcher +type Event struct { + Type EventType + BlockHeader *MiniHeader +} + +// Config holds some configuration options for an instance of BlockWatcher. +type Config struct { + Store MiniHeaderStore + PollingInterval time.Duration + StartBlockDepth rpc.BlockNumber + BlockRetentionLimit int + WithLogs bool + Topics []common.Hash + Client Client +} + +// Watcher maintains a consistent representation of the latest `blockRetentionLimit` blocks, +// handling block re-orgs and network disruptions gracefully. It can be started from any arbitrary +// block height, and will emit both block added and removed events. +type Watcher struct { + blockRetentionLimit int + startBlockDepth rpc.BlockNumber + stack *Stack + client Client + blockFeed event.Feed + blockScope event.SubscriptionScope // Subscription scope tracking current live listeners + wasStartedOnce bool // Whether the block watcher has previously been started + pollingInterval time.Duration + ticker *time.Ticker + withLogs bool + topics []common.Hash + mu sync.RWMutex +} + +// New creates a new Watcher instance. +func New(config Config) *Watcher { + stack := NewStack(config.Store, config.BlockRetentionLimit) + + bs := &Watcher{ + pollingInterval: config.PollingInterval, + blockRetentionLimit: config.BlockRetentionLimit, + startBlockDepth: config.StartBlockDepth, + stack: stack, + client: config.Client, + withLogs: config.WithLogs, + topics: config.Topics, + } + return bs +} + +// BackfillEventsIfNeeded finds missed events that might have occured while the +// node was offline and sends them to event subscribers. It blocks until +// it is done backfilling or the given context is canceled. +func (w *Watcher) BackfillEventsIfNeeded(ctx context.Context) error { + events, err := w.getMissedEventsToBackfill(ctx) + if err != nil { + return err + } + if len(events) > 0 { + w.blockFeed.Send(events) + } + return nil +} + +// Watch starts the Watcher. It will continuously look for new blocks and blocks +// until the given context is canceled. Typically, you want to call Watch inside a goroutine. +func (w *Watcher) Watch(ctx context.Context) error { + w.mu.Lock() + if w.wasStartedOnce { + w.mu.Unlock() + return errors.New("Can only start Watcher once per instance") + } + w.wasStartedOnce = true + w.mu.Unlock() + + ticker := time.NewTicker(w.pollingInterval) + for { + select { + case <-ctx.Done(): + ticker.Stop() + return nil + case <-ticker.C: + if err := w.pollNextBlock(); err != nil { + glog.Error("blockwatch.Watcher error encountered") + } + } + } +} + +// Subscribe allows one to subscribe to the block events emitted by the Watcher. +// To unsubscribe, simply call `Unsubscribe` on the returned subscription. +// The sink channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. +func (w *Watcher) Subscribe(sink chan<- []*Event) event.Subscription { + return w.blockScope.Track(w.blockFeed.Subscribe(sink)) +} + +// GetLatestBlock returns the latest block processed +func (w *Watcher) GetLatestBlock() (*MiniHeader, error) { + return w.stack.Peek() +} + +// InspectRetainedBlocks returns the blocks retained in-memory by the Watcher instance. It is not +// particularly performant and therefore should only be used for debugging and testing purposes. +func (w *Watcher) InspectRetainedBlocks() ([]*MiniHeader, error) { + return w.stack.Inspect() +} + +// pollNextBlock polls for the next block header to be added to the block stack. +// If there are no blocks on the stack, it fetches the first block at the specified +// `startBlockDepth` supplied at instantiation. +func (w *Watcher) pollNextBlock() error { + var nextBlockNumber *big.Int + latestHeader, err := w.stack.Peek() + if err != nil { + return err + } + if latestHeader == nil { + if w.startBlockDepth == rpc.LatestBlockNumber { + nextBlockNumber = nil // Fetch latest block + } else { + nextBlockNumber = big.NewInt(int64(w.startBlockDepth)) + } + } else { + nextBlockNumber = big.NewInt(0).Add(latestHeader.Number, big.NewInt(1)) + } + nextHeader, err := w.client.HeaderByNumber(nextBlockNumber) + if err != nil { + if err == ethereum.NotFound { + glog.Infof("block header not found blockNumber=%v", nextBlockNumber) + return nil // Noop and wait next polling interval + } + return err + } + + events := []*Event{} + events, err = w.buildCanonicalChain(nextHeader, events) + // Even if an error occurred, we still want to emit the events gathered since we might have + // popped blocks off the Stack and they won't be re-added + if len(events) != 0 { + w.blockFeed.Send(events) + } + if err != nil { + return err + } + return nil +} + +func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ([]*Event, error) { + latestHeader, err := w.stack.Peek() + if err != nil { + return nil, err + } + // Is the stack empty or is it the next block? + if latestHeader == nil || nextHeader.Parent == latestHeader.Hash { + nextHeader, err := w.addLogs(nextHeader) + if err != nil { + // Due to block re-orgs & Ethereum node services load-balancing requests across multiple nodes + // a block header might be returned, but when fetching it's logs, an "unknown block" error is + // returned. This is expected to happen sometimes, and we simply return the events gathered so + // far and pick back up where we left off on the next polling interval. + if isUnknownBlockErr(err) { + glog.Infof("failed to get logs for block nextHeader=%v", nextHeader) + return events, nil + } + return events, err + } + err = w.stack.Push(nextHeader) + if err != nil { + return events, err + } + events = append(events, &Event{ + Type: Added, + BlockHeader: nextHeader, + }) + return events, nil + } + + // Pop latestHeader from the stack. We already have a reference to it. + if _, err := w.stack.Pop(); err != nil { + return events, err + } + events = append(events, &Event{ + Type: Removed, + BlockHeader: latestHeader, + }) + + nextParentHeader, err := w.client.HeaderByHash(nextHeader.Parent) + if err != nil { + if err == ethereum.NotFound { + glog.Infof("block header not found blockHash=%v", nextHeader.Parent.Hex()) + // Noop and wait next polling interval. We remove the popped blocks + // and refetch them on the next polling interval. + return events, nil + } + return events, err + } + events, err = w.buildCanonicalChain(nextParentHeader, events) + if err != nil { + return events, err + } + nextHeader, err = w.addLogs(nextHeader) + if err != nil { + // Due to block re-orgs & Ethereum node services load-balancing requests across multiple nodes + // a block header might be returned, but when fetching it's logs, an "unknown block" error is + // returned. This is expected to happen sometimes, and we simply return the events gathered so + // far and pick back up where we left off on the next polling interval. + if isUnknownBlockErr(err) { + glog.Infof("failed to get logs for block nextHeader=%v", nextHeader) + return events, nil + } + return events, err + } + err = w.stack.Push(nextHeader) + if err != nil { + return events, err + } + events = append(events, &Event{ + Type: Added, + BlockHeader: nextHeader, + }) + + return events, nil +} + +func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { + if !w.withLogs { + return header, nil + } + logs, err := w.client.FilterLogs(ethereum.FilterQuery{ + BlockHash: &header.Hash, + Topics: [][]common.Hash{w.topics}, + }) + if err != nil { + return header, err + } + header.Logs = logs + return header, nil +} + +// getMissedEventsToBackfill finds missed events that might have occured while the node was +// offline. It does this by comparing the last block stored with the latest block discoverable via RPC. +// If the stored block is older then the latest block, it batch fetches the events for missing blocks, +// re-sets the stored blocks and returns the block events found. +func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, error) { + events := []*Event{} + + latestRetainedBlock, err := w.stack.Peek() + if err != nil { + return events, err + } + // No blocks stored, nowhere to backfill to + if latestRetainedBlock == nil { + return events, nil + } + latestBlock, err := w.client.HeaderByNumber(nil) + if err != nil { + return events, err + } + blocksElapsed := big.NewInt(0).Sub(latestBlock.Number, latestRetainedBlock.Number) + if blocksElapsed.Int64() == 0 { + return events, nil + } + + glog.Infof("Some blocks have elapsed since last boot. Backfilling block events (this can take a while)... blocksElapsed=%v", blocksElapsed.Int64()) + startBlockNum := int(latestRetainedBlock.Number.Int64() + 1) + endBlockNum := int(latestRetainedBlock.Number.Int64() + blocksElapsed.Int64()) + logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, endBlockNum) + if int64(furthestBlockProcessed) > latestRetainedBlock.Number.Int64() { + // If we have processed blocks further then the latestRetainedBlock in the DB, we + // want to remove all blocks from the DB and insert the furthestBlockProcessed + // Doing so will cause the BlockWatcher to start from that furthestBlockProcessed. + headers, err := w.InspectRetainedBlocks() + if err != nil { + return events, err + } + for i := 0; i < len(headers); i++ { + _, err := w.stack.Pop() + if err != nil { + return events, err + } + } + // Add furthest block processed into the DB + latestHeader, err := w.client.HeaderByNumber(big.NewInt(int64(furthestBlockProcessed))) + if err != nil { + return events, err + } + err = w.stack.Push(latestHeader) + if err != nil { + return events, err + } + + // If no logs found, noop + if len(logs) == 0 { + return events, nil + } + + // Create the block events from all the logs found by grouping + // them into blockHeaders + hashToBlockHeader := map[common.Hash]*MiniHeader{} + for _, log := range logs { + blockHeader, ok := hashToBlockHeader[log.BlockHash] + if !ok { + // TODO(fabio): Find a way to include the parent hash for the block as well. + // It's currently not an issue to omit it since we don't use the parent hash + // when processing block events in OrderWatcher. + blockHeader = &MiniHeader{ + Hash: log.BlockHash, + Number: big.NewInt(0).SetUint64(log.BlockNumber), + Logs: []types.Log{}, + } + hashToBlockHeader[log.BlockHash] = blockHeader + } + blockHeader.Logs = append(blockHeader.Logs, log) + } + for _, blockHeader := range hashToBlockHeader { + events = append(events, &Event{ + Type: Added, + BlockHeader: blockHeader, + }) + } + glog.Info("Done backfilling block events") + return events, nil + } + return events, nil +} + +type logRequestResult struct { + From int + To int + Logs []types.Log + Err error +} + +// getLogsRequestChunkSize is the number of `eth_getLogs` JSON RPC to send concurrently in each batch fetch +const getLogsRequestChunkSize = 3 + +// getLogsInBlockRange attempts to fetch all logs in the block range supplied. It implements a +// limited-concurrency batch fetch, where all requests in the previous batch must complete for +// the next batch of requests to be sent. If an error is encountered in a batch, all subsequent +// batch requests are not sent. Instead, it returns all the logs it found up until the error was +// encountered, along with the block number after which no further logs were retrieved. +func (w *Watcher) getLogsInBlockRange(ctx context.Context, from, to int) ([]types.Log, int) { + blockRanges := w.getSubBlockRanges(from, to, maxBlocksInGetLogsQuery) + + numChunks := 0 + chunkChan := make(chan []*blockRange, 1000000) + for len(blockRanges) != 0 { + var chunk []*blockRange + if len(blockRanges) < getLogsRequestChunkSize { + chunk = blockRanges[:len(blockRanges)] + } else { + chunk = blockRanges[:getLogsRequestChunkSize] + } + chunkChan <- chunk + blockRanges = blockRanges[len(chunk):] + numChunks++ + } + + semaphoreChan := make(chan struct{}, 1) + defer close(semaphoreChan) + + didAPreviousRequestFail := false + furthestBlockProcessed := from - 1 + allLogs := []types.Log{} + + for i := 0; i < numChunks; i++ { + // Add one to the semaphore chan. If it already has a value, the chunk blocks here until one frees up. + // We deliberately process the chunks sequentially, since if any request results in an error, we + // do not want to send any further requests. + semaphoreChan <- struct{}{} + + // If a previous request failed, we stop processing newer requests + if didAPreviousRequestFail { + <-semaphoreChan + continue // Noop + } + + mu := sync.Mutex{} + indexToLogResult := map[int]logRequestResult{} + chunk := <-chunkChan + + wg := &sync.WaitGroup{} + for i, aBlockRange := range chunk { + wg.Add(1) + go func(index int, b *blockRange) { + defer wg.Done() + + select { + case <-ctx.Done(): + indexToLogResult[index] = logRequestResult{ + From: b.FromBlock, + To: b.ToBlock, + Err: errors.New("context was canceled"), + Logs: []types.Log{}, + } + return + default: + } + + logs, err := w.filterLogsRecursively(b.FromBlock, b.ToBlock, []types.Log{}) + if err != nil { + glog.Errorf("failed to fetch logs for range error=%v fromBlock=%v toBlock=%v", err, b.FromBlock, b.ToBlock) + } + mu.Lock() + indexToLogResult[index] = logRequestResult{ + From: b.FromBlock, + To: b.ToBlock, + Err: err, + Logs: logs, + } + mu.Unlock() + }(i, aBlockRange) + } + + // Wait for all log requests to complete + wg.Wait() + + for i, aBlockRange := range chunk { + logRequestResult := indexToLogResult[i] + // Break at first error encountered + if logRequestResult.Err != nil { + didAPreviousRequestFail = true + furthestBlockProcessed = logRequestResult.From - 1 + break + } + allLogs = append(allLogs, logRequestResult.Logs...) + furthestBlockProcessed = aBlockRange.ToBlock + } + <-semaphoreChan + } + + return allLogs, furthestBlockProcessed +} + +type blockRange struct { + FromBlock int + ToBlock int +} + +// getSubBlockRanges breaks up the block range into smaller block ranges of rangeSize. +// `eth_getLogs` requests are inclusive to both the start and end blocks specified and +// so we need to make the ranges exclusive of one another to avoid fetching the same +// blocks' logs twice. +func (w *Watcher) getSubBlockRanges(from, to, rangeSize int) []*blockRange { + chunks := []*blockRange{} + numBlocksLeft := to - from + if numBlocksLeft < rangeSize { + chunks = append(chunks, &blockRange{ + FromBlock: from, + ToBlock: to, + }) + } else { + blocks := []int{} + for i := 0; i <= numBlocksLeft; i++ { + blocks = append(blocks, from+i) + } + numChunks := len(blocks) / rangeSize + remainder := len(blocks) % rangeSize + if remainder > 0 { + numChunks = numChunks + 1 + } + + for i := 0; i < numChunks; i = i + 1 { + fromIndex := i * rangeSize + toIndex := fromIndex + rangeSize + if toIndex > len(blocks) { + toIndex = len(blocks) + } + bs := blocks[fromIndex:toIndex] + blockRange := &blockRange{ + FromBlock: bs[0], + ToBlock: bs[len(bs)-1], + } + chunks = append(chunks, blockRange) + } + } + return chunks +} + +const infuraTooManyResultsErrMsg = "query returned more than 10000 results" + +func (w *Watcher) filterLogsRecursively(from, to int, allLogs []types.Log) ([]types.Log, error) { + glog.Infof("fetching block logs from=%v to=%v", from, to) + numBlocks := to - from + topics := [][]common.Hash{} + if len(w.topics) > 0 { + topics = append(topics, w.topics) + } + logs, err := w.client.FilterLogs(ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(from)), + ToBlock: big.NewInt(int64(to)), + Topics: topics, + }) + if err != nil { + // Infura caps the logs returned to 10,000 per request, if our request exceeds this limit, split it + // into two requests. Parity, Geth and Alchemy all have much higher limits (if any at all), so no need + // to expect any similar errors of this nature from them. + if err.Error() == infuraTooManyResultsErrMsg { + // HACK(fabio): Infura limits the returned results to 10,000 logs, BUT some single + // blocks contain more then 10,000 logs. This has supposedly been fixed but we keep + // this logic here just in case. It helps us avoid infinite recursion. + // Source: https://community.infura.io/t/getlogs-error-query-returned-more-than-1000-results/358/10 + if from == to { + return allLogs, fmt.Errorf("Unable to get the logs for block #%d, because it contains too many logs", from) + } + + r := numBlocks % 2 + firstBatchSize := numBlocks / 2 + secondBatchSize := firstBatchSize + if r == 1 { + secondBatchSize = secondBatchSize + 1 + } + + endFirstHalf := from + firstBatchSize + startSecondHalf := endFirstHalf + 1 + allLogs, err := w.filterLogsRecursively(from, endFirstHalf, allLogs) + if err != nil { + return nil, err + } + allLogs, err = w.filterLogsRecursively(startSecondHalf, to, allLogs) + if err != nil { + return nil, err + } + return allLogs, nil + } else { + return nil, err + } + } + allLogs = append(allLogs, logs...) + return allLogs, nil +} + +func isUnknownBlockErr(err error) bool { + // Geth error + if err.Error() == "unknown block" { + return true + } + // Parity error + if err.Error() == "One of the blocks specified in filter (fromBlock, toBlock or blockHash) cannot be found" { + return true + } + return false +} diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go new file mode 100644 index 0000000..4f11787 --- /dev/null +++ b/eth/blockwatch/block_watcher_test.go @@ -0,0 +1,487 @@ +package blockwatch + +import ( + "context" + "errors" + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var config = Config{ + PollingInterval: 1 * time.Second, + BlockRetentionLimit: 10, + StartBlockDepth: rpc.LatestBlockNumber, + WithLogs: false, + Topics: []common.Hash{}, +} + +var basicFakeClientFixture = "testdata/fake_client_basic_fixture.json" + +func TestWatcher(t *testing.T) { + fakeClient, err := newFakeClient("testdata/fake_client_block_poller_fixtures.json") + require.NoError(t, err) + + // Polling interval unused because we hijack the ticker for this test + config.Store = &stubMiniHeaderStore{} + config.Client = fakeClient + watcher := New(config) + + // Having a buffer of 1 unblocks the below for-loop without resorting to a goroutine + events := make(chan []*Event, 1) + sub := watcher.Subscribe(events) + + for i := 0; i < fakeClient.NumberOfTimesteps(); i++ { + scenarioLabel := fakeClient.GetScenarioLabel() + + err := watcher.pollNextBlock() + require.NoError(t, err) + + retainedBlocks, err := watcher.InspectRetainedBlocks() + require.NoError(t, err) + expectedRetainedBlocks := fakeClient.ExpectedRetainedBlocks() + assert.Equal(t, expectedRetainedBlocks, retainedBlocks, scenarioLabel) + + expectedEvents := fakeClient.GetEvents() + if len(expectedEvents) != 0 { + select { + case gotEvents := <-events: + assert.Equal(t, expectedEvents, gotEvents, scenarioLabel) + + case <-time.After(3 * time.Second): + t.Fatal("Timed out waiting for Events channel to deliver expected events") + } + } + + fakeClient.IncrementTimestep() + + if i == fakeClient.NumberOfTimesteps()-1 { + sub.Unsubscribe() + } + } +} + +func TestWatcherStartStop(t *testing.T) { + fakeClient, err := newFakeClient(basicFakeClientFixture) + require.NoError(t, err) + + config.Store = &stubMiniHeaderStore{} + config.Client = fakeClient + watcher := New(config) + + // Start the watcher in a goroutine. We use the done channel to signal when + // watcher.Watch returns. + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + defer cancel() + go func() { + require.NoError(t, watcher.Watch(ctx)) + done <- struct{}{} + }() + + // Wait a bit and then stop the watcher by calling cancel. + time.Sleep(100 * time.Millisecond) + cancel() + + // Make sure that the watcher actually stops. + select { + case <-time.After(100 * time.Millisecond): + t.Error("timed out waiting for watcher to stop") + case <-done: + break + } +} + +type blockRangeChunksTestCase struct { + from int + to int + expectedBlockRanges []*blockRange +} + +func TestGetSubBlockRanges(t *testing.T) { + rangeSize := 6 + testCases := []blockRangeChunksTestCase{ + blockRangeChunksTestCase{ + from: 10, + to: 10, + expectedBlockRanges: []*blockRange{ + &blockRange{ + FromBlock: 10, + ToBlock: 10, + }, + }, + }, + blockRangeChunksTestCase{ + from: 10, + to: 16, + expectedBlockRanges: []*blockRange{ + &blockRange{ + FromBlock: 10, + ToBlock: 15, + }, + &blockRange{ + FromBlock: 16, + ToBlock: 16, + }, + }, + }, + blockRangeChunksTestCase{ + from: 10, + to: 22, + expectedBlockRanges: []*blockRange{ + &blockRange{ + FromBlock: 10, + ToBlock: 15, + }, + &blockRange{ + FromBlock: 16, + ToBlock: 21, + }, + &blockRange{ + FromBlock: 22, + ToBlock: 22, + }, + }, + }, + blockRangeChunksTestCase{ + from: 10, + to: 24, + expectedBlockRanges: []*blockRange{ + &blockRange{ + FromBlock: 10, + ToBlock: 15, + }, + &blockRange{ + FromBlock: 16, + ToBlock: 21, + }, + &blockRange{ + FromBlock: 22, + ToBlock: 24, + }, + }, + }, + } + + fakeClient, err := newFakeClient(basicFakeClientFixture) + require.NoError(t, err) + config.Store = &stubMiniHeaderStore{} + config.Client = fakeClient + watcher := New(config) + + for _, testCase := range testCases { + blockRanges := watcher.getSubBlockRanges(testCase.from, testCase.to, rangeSize) + assert.Equal(t, testCase.expectedBlockRanges, blockRanges) + } +} + +func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { + // Fixture will return block 30 as the tip of the chain + fakeClient, err := newFakeClient("testdata/fake_client_fast_sync_fixture.json") + require.NoError(t, err) + + store := &stubMiniHeaderStore{} + // Add block number 5 as the last block seen by BlockWatcher + lastBlockSeen := &MiniHeader{ + Number: big.NewInt(5), + Hash: common.HexToHash("0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f"), + Parent: common.HexToHash("0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba"), + } + err = store.InsertMiniHeader(lastBlockSeen) + require.NoError(t, err) + + config.Store = store + config.Client = fakeClient + watcher := New(config) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events, err := watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 1) + + // Check that block 30 is now in the DB, and block 5 was removed. + headers, err := store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 1) + assert.Equal(t, big.NewInt(30), headers[0].Number) +} + +func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { + // Fixture will return block 5 as the tip of the chain + fakeClient, err := newFakeClient("testdata/fake_client_basic_fixture.json") + require.NoError(t, err) + + store := &stubMiniHeaderStore{} + // Add block number 5 as the last block seen by BlockWatcher + lastBlockSeen := &MiniHeader{ + Number: big.NewInt(5), + Hash: common.HexToHash("0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f"), + Parent: common.HexToHash("0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba"), + } + err = store.InsertMiniHeader(lastBlockSeen) + require.NoError(t, err) + + config.Store = store + config.Client = fakeClient + watcher := New(config) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events, err := watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 0) + + // Check that block 5 is still in the DB + headers, err := store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 1) + assert.Equal(t, big.NewInt(5), headers[0].Number) +} + +var logStub = types.Log{ + Address: common.HexToAddress("0x21ab6c9fac80c59d401b37cb43f81ea9dde7fe34"), + Topics: []common.Hash{ + common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), + common.HexToHash("0x0000000000000000000000004d8a4aa1f304f9632cf3877473445d85c577fe5d"), + common.HexToHash("0x0000000000000000000000004bdd0d16cfa18e33860470fc4d65c6f5cee60959"), + }, + Data: common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000337ad34c0"), + BlockNumber: uint64(30), + TxHash: common.HexToHash("0xd9bb5f9e888ee6f74bedcda811c2461230f247c205849d6f83cb6c3925e54586"), + TxIndex: uint(0), + BlockHash: common.HexToHash("0x6bbf9b6e836207ab25379c20e517a89090cbbaf8877746f6ed7fb6820770816b"), + Index: uint(0), + Removed: false, +} + +var errUnexpected = errors.New("Something unexpected") + +type filterLogsRecursivelyTestCase struct { + Label string + rangeToFilterLogsResponse map[string]filterLogsResponse + Err error + Logs []types.Log +} + +func TestFilterLogsRecursively(t *testing.T) { + from := 10 + to := 20 + testCases := []filterLogsRecursivelyTestCase{ + filterLogsRecursivelyTestCase{ + Label: "HAPPY_PATH", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub}, + }, + filterLogsRecursivelyTestCase{ + Label: "TOO_MANY_RESULTS_INFURA_ERROR", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Err: errors.New(infuraTooManyResultsErrMsg), + }, + "10-15": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + "16-20": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub, logStub}, + }, + filterLogsRecursivelyTestCase{ + Label: "TOO_MANY_RESULTS_INFURA_ERROR_DEEPER_RECURSION", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Err: errors.New(infuraTooManyResultsErrMsg), + }, + "10-15": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + "16-20": filterLogsResponse{ + Err: errors.New(infuraTooManyResultsErrMsg), + }, + "16-18": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + "19-20": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub, logStub, logStub}, + }, + filterLogsRecursivelyTestCase{ + Label: "TOO_MANY_RESULTS_INFURA_ERROR_DEEPER_RECURSION_FAILURE", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Err: errors.New(infuraTooManyResultsErrMsg), + }, + "10-15": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + "16-20": filterLogsResponse{ + Err: errUnexpected, + }, + }, + Err: errUnexpected, + }, + filterLogsRecursivelyTestCase{ + Label: "UNEXPECTED_ERROR", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Err: errUnexpected, + }, + }, + Err: errUnexpected, + }, + } + + config.Store = &stubMiniHeaderStore{} + + for _, testCase := range testCases { + fakeLogClient, err := newFakeLogClient(testCase.rangeToFilterLogsResponse) + require.NoError(t, err) + config.Client = fakeLogClient + watcher := New(config) + + logs, err := watcher.filterLogsRecursively(from, to, []types.Log{}) + require.Equal(t, testCase.Err, err, testCase.Label) + require.Equal(t, testCase.Logs, logs, testCase.Label) + assert.Equal(t, len(testCase.rangeToFilterLogsResponse), fakeLogClient.Count()) + } +} + +type logsInBlockRangeTestCase struct { + Label string + From int + To int + RangeToFilterLogsResponse map[string]filterLogsResponse + Logs []types.Log + FurthestBlockProcessed int +} + +func TestGetLogsInBlockRange(t *testing.T) { + from := 10 + to := 20 + testCases := []logsInBlockRangeTestCase{ + logsInBlockRangeTestCase{ + Label: "HAPPY_PATH", + From: from, + To: to, + RangeToFilterLogsResponse: map[string]filterLogsResponse{ + aRange(from, to): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub}, + FurthestBlockProcessed: to, + }, + logsInBlockRangeTestCase{ + Label: "SPLIT_REQUEST_BY_MAX_BLOCKS_IN_QUERY", + From: from, + To: from + maxBlocksInGetLogsQuery + 10, + RangeToFilterLogsResponse: map[string]filterLogsResponse{ + aRange(from, from+maxBlocksInGetLogsQuery-1): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + aRange(from+maxBlocksInGetLogsQuery, from+maxBlocksInGetLogsQuery+10): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub, logStub}, + FurthestBlockProcessed: from + maxBlocksInGetLogsQuery + 10, + }, + logsInBlockRangeTestCase{ + Label: "SHORT_CIRCUIT_SEMAPHORE_BLOCKED_REQUESTS_ON_ERROR", + From: from, + To: from + (maxBlocksInGetLogsQuery * (getLogsRequestChunkSize + 1)), + RangeToFilterLogsResponse: map[string]filterLogsResponse{ + // Same number of responses as the getLogsRequestChunkSize since the + // error response will stop any further requests. + aRange(from, from+maxBlocksInGetLogsQuery-1): filterLogsResponse{ + Err: errUnexpected, + }, + aRange(from+maxBlocksInGetLogsQuery, from+(maxBlocksInGetLogsQuery*2)-1): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + aRange(from+(maxBlocksInGetLogsQuery*2), from+(maxBlocksInGetLogsQuery*3)-1): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{}, + FurthestBlockProcessed: from - 1, + }, + logsInBlockRangeTestCase{ + Label: "CORRECT_FURTHEST_BLOCK_PROCESSED_ON_ERROR", + From: from, + To: from + maxBlocksInGetLogsQuery + 10, + RangeToFilterLogsResponse: map[string]filterLogsResponse{ + aRange(from, from+maxBlocksInGetLogsQuery-1): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + aRange(from+maxBlocksInGetLogsQuery, from+maxBlocksInGetLogsQuery+10): filterLogsResponse{ + Err: errUnexpected, + }}, + Logs: []types.Log{logStub}, + FurthestBlockProcessed: from + maxBlocksInGetLogsQuery - 1, + }, + } + + config.Store = &stubMiniHeaderStore{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for _, testCase := range testCases { + fakeLogClient, err := newFakeLogClient(testCase.RangeToFilterLogsResponse) + require.NoError(t, err) + config.Client = fakeLogClient + watcher := New(config) + + logs, furthestBlockProcessed := watcher.getLogsInBlockRange(ctx, testCase.From, testCase.To) + require.Equal(t, testCase.FurthestBlockProcessed, furthestBlockProcessed, testCase.Label) + require.Equal(t, testCase.Logs, logs, testCase.Label) + assert.Equal(t, len(testCase.RangeToFilterLogsResponse), fakeLogClient.Count()) + } +} + +func aRange(from, to int) string { + r := fmt.Sprintf("%d-%d", from, to) + return r +} From cf16c6b94e262d5413d251c0a29942be32e07b88 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 15 Aug 2019 16:00:04 +0200 Subject: [PATCH 02/72] eth: roundswatcher with cache Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: eb28eb338ab84d4eae7611b4d530a0ea5be98797 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 151 +++++++++++++++++++++++++++++ eth/watchers/roundswatcher_test.go | 99 +++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 eth/watchers/roundswatcher.go create mode 100644 eth/watchers/roundswatcher_test.go diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go new file mode 100644 index 0000000..0029aee --- /dev/null +++ b/eth/watchers/roundswatcher.go @@ -0,0 +1,151 @@ +package watchers + +import ( + "fmt" + "log" + "math/big" + "sync" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/go-livepeer/eth/contracts" + + "github.com/golang/glog" + "github.com/livepeer/go-livepeer/eth" + "github.com/livepeer/go-livepeer/eth/blockwatch" +) + +// RoundsWatcher is a type for a thread safe in-memory cache that watches on-chain events for new initialized rounds +type RoundsWatcher struct { + mu sync.RWMutex + lastInitializedRound *big.Int + lastInitializedBlockHash [32]byte + + quit chan struct{} + + watcher BlockWatcher + lpEth eth.LivepeerEthClient + dec *EventDecoder +} + +// NewRoundsWatcher creates a new instance of RoundsWatcher and sets the initial cache through an RPC call to an ethereum node +func NewRoundsWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, lpEth eth.LivepeerEthClient) (*RoundsWatcher, error) { + dec, err := NewEventDecoder(roundsManagerAddr, contracts.RoundsManagerABI) + if err != nil { + return nil, err + } + lr, err := lpEth.LastInitializedRound() + if err != nil { + return nil, err + } + bh, err := lpEth.BlockHashForRound(lr) + if err != nil { + return nil, err + } + + return &RoundsWatcher{ + quit: make(chan struct{}), + watcher: watcher, + lpEth: lpEth, + lastInitializedRound: lr, + lastInitializedBlockHash: bh, + dec: dec, + }, nil +} + +// LastInitializedRound gets the last initialized round from cache +func (rw *RoundsWatcher) LastInitializedRound() *big.Int { + rw.mu.RLock() + defer rw.mu.RUnlock() + return rw.lastInitializedRound +} + +// LastInitializedBlockHash returns the blockhash of the block the last round was initiated in +func (rw *RoundsWatcher) LastInitializedBlockHash() [32]byte { + rw.mu.RLock() + defer rw.mu.RUnlock() + return rw.lastInitializedBlockHash +} + +func (rw *RoundsWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) { + rw.mu.Lock() + defer rw.mu.Unlock() + rw.lastInitializedRound = round + rw.lastInitializedBlockHash = hash +} + +// Watch the blockwatch subscription for NewRound events +func (rw *RoundsWatcher) Watch() { + lr, err := rw.lpEth.LastInitializedRound() + if err != nil { + glog.Errorf("error fetching initial lastInitializedRound value: %v", err) + } + + bh, err := rw.lpEth.BlockHashForRound(lr) + if err != nil { + glog.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) + } + rw.setLastInitializedRound(lr, bh) + + events := make(chan []*blockwatch.Event, 10) + sub := rw.watcher.Subscribe(events) + defer sub.Unsubscribe() + for { + select { + case <-rw.quit: + return + case err := <-sub.Err(): + log.Fatal(err) + case events := <-events: + rw.handleBlockEvents(events) + } + } +} + +// Stop watching for NewRound events +func (rw *RoundsWatcher) Stop() { + close(rw.quit) +} + +func (rw *RoundsWatcher) handleBlockEvents(events []*blockwatch.Event) { + for _, event := range events { + for _, log := range event.BlockHeader.Logs { + if event.Type == blockwatch.Removed { + log.Removed = true + } + if err := rw.handleLog(log); err != nil { + glog.Error(err) + } + } + } +} + +func (rw *RoundsWatcher) handleLog(log types.Log) error { + eventName, err := rw.dec.FindEventName(log) + if err != nil { + return fmt.Errorf("could not find event name: %v", err) + } + + if eventName != "NewRound" { + return fmt.Errorf("eventName is not NewRound") + } + + var nr contracts.RoundsManagerNewRound + if err := rw.dec.Decode("NewRound", log, &nr); err != nil { + return fmt.Errorf("unable to decode event: %v", err) + } + if log.Removed { + lr, err := rw.lpEth.LastInitializedRound() + if err != nil { + return err + } + bh, err := rw.lpEth.BlockHashForRound(lr) + if err != nil { + return err + } + rw.setLastInitializedRound(lr, bh) + } else { + rw.setLastInitializedRound(nr.Round, nr.BlockHash) + } + return nil +} diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go new file mode 100644 index 0000000..78dce2a --- /dev/null +++ b/eth/watchers/roundswatcher_test.go @@ -0,0 +1,99 @@ +package watchers + +import ( + "math/big" + "testing" + "time" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/go-livepeer/eth" + "github.com/livepeer/go-livepeer/eth/blockwatch" + "github.com/livepeer/go-livepeer/pm" + "github.com/stretchr/testify/assert" +) + +func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { + assert := assert.New(t) + rw := &RoundsWatcher{} + round := big.NewInt(5) + var hash [32]byte + copy(hash[:], "hello world") + rw.setLastInitializedRound(round, hash) + assert.Equal(rw.lastInitializedRound, round) + assert.Equal(rw.lastInitializedBlockHash, hash) + + r := rw.LastInitializedRound() + assert.Equal(r, round) + h := rw.LastInitializedBlockHash() + assert.Equal(h, hash) +} + +func TestWatchAndStop(t *testing.T) { + assert := assert.New(t) + lpEth := ð.StubClient{} + watcher := &stubBlockWatcher{} + rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(err) + + header := defaultMiniHeader() + newRoundEvent := newStubNewRoundLog() + + header.Logs = append(header.Logs, newRoundEvent) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + go rw.Watch() + time.Sleep(2 * time.Millisecond) + + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound := rw.LastInitializedRound() + assert.Zero(lastRound.Cmp(big.NewInt(8))) + bhForRound := rw.LastInitializedBlockHash() + var expectedHashForRound [32]byte + copy(expectedHashForRound[:], newRoundEvent.Data[:]) + assert.Equal(bhForRound, expectedHashForRound) + + // Test no NewRound events, values on rw remain the same + blockEvent.BlockHeader.Logs = header.Logs[:1] + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound = rw.LastInitializedRound() + assert.Zero(lastRound.Cmp(big.NewInt(8))) + bhForRound = rw.LastInitializedBlockHash() + copy(expectedHashForRound[:], newRoundEvent.Data[:]) + assert.Equal(bhForRound, expectedHashForRound) + + // Test RPC paths (event removed) + blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) + blockEvent.Type = blockwatch.Removed + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound = rw.LastInitializedRound() + bhForRound = rw.LastInitializedBlockHash() + assert.Equal(lastRound.Int64(), int64(0)) + assert.Equal(bhForRound, [32]byte{}) + + // Test Stop + rw.Stop() + time.Sleep(2 * time.Millisecond) + assert.True(watcher.sub.unsubscribed) +} + +func defaultMiniHeader() *blockwatch.MiniHeader { + block := &blockwatch.MiniHeader{ + Number: big.NewInt(450), + Parent: pm.RandHash(), + Hash: pm.RandHash(), + } + log := types.Log{ + Topics: []ethcommon.Hash{pm.RandHash(), pm.RandHash()}, + Data: pm.RandBytes(32), + BlockHash: block.Hash, + } + block.Logs = []types.Log{log} + return block +} From f894b052007543bfd5cd9a62f1cf222d78d4ba80 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 19 Aug 2019 14:41:23 +0200 Subject: [PATCH 03/72] cmd, eth: init roundswatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 84c13f8b82cf4103ea55e7088304e1ead78c6156 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 0029aee..efaa4e6 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -2,7 +2,6 @@ package watchers import ( "fmt" - "log" "math/big" "sync" @@ -32,24 +31,14 @@ type RoundsWatcher struct { func NewRoundsWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, lpEth eth.LivepeerEthClient) (*RoundsWatcher, error) { dec, err := NewEventDecoder(roundsManagerAddr, contracts.RoundsManagerABI) if err != nil { - return nil, err - } - lr, err := lpEth.LastInitializedRound() - if err != nil { - return nil, err - } - bh, err := lpEth.BlockHashForRound(lr) - if err != nil { - return nil, err + return nil, fmt.Errorf("error creating decoder: %v", err) } return &RoundsWatcher{ - quit: make(chan struct{}), - watcher: watcher, - lpEth: lpEth, - lastInitializedRound: lr, - lastInitializedBlockHash: bh, - dec: dec, + quit: make(chan struct{}), + watcher: watcher, + lpEth: lpEth, + dec: dec, }, nil } @@ -80,7 +69,6 @@ func (rw *RoundsWatcher) Watch() { if err != nil { glog.Errorf("error fetching initial lastInitializedRound value: %v", err) } - bh, err := rw.lpEth.BlockHashForRound(lr) if err != nil { glog.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) @@ -95,7 +83,7 @@ func (rw *RoundsWatcher) Watch() { case <-rw.quit: return case err := <-sub.Err(): - log.Fatal(err) + glog.Error(err) case events := <-events: rw.handleBlockEvents(events) } From dedca186dcaf1ea30571a4607e638aa09993793f Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Mon, 26 Aug 2019 16:41:36 -0400 Subject: [PATCH 04/72] eth/watchers: Return nil for unknown events in RoundsWatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 7fefbe878d4540f52c02177ae7b303dae96cf74c Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 3 ++- eth/watchers/roundswatcher_test.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index efaa4e6..20031bd 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -111,7 +111,8 @@ func (rw *RoundsWatcher) handleBlockEvents(events []*blockwatch.Event) { func (rw *RoundsWatcher) handleLog(log types.Log) error { eventName, err := rw.dec.FindEventName(log) if err != nil { - return fmt.Errorf("could not find event name: %v", err) + // Noop if we cannot find the event name + return nil } if eventName != "NewRound" { diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go index 78dce2a..9ead20c 100644 --- a/eth/watchers/roundswatcher_test.go +++ b/eth/watchers/roundswatcher_test.go @@ -11,6 +11,7 @@ import ( "github.com/livepeer/go-livepeer/eth/blockwatch" "github.com/livepeer/go-livepeer/pm" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { @@ -83,6 +84,24 @@ func TestWatchAndStop(t *testing.T) { assert.True(watcher.sub.unsubscribed) } +func TestRoundsWatcher_HandleLog(t *testing.T) { + lpEth := ð.StubClient{} + watcher := &stubBlockWatcher{} + rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) + require.Nil(t, err) + + assert := assert.New(t) + + // Test unknown event + log := newStubBaseLog() + log.Topics = []ethcommon.Hash{ethcommon.BytesToHash([]byte("foo"))} + + err = rw.handleLog(log) + assert.Nil(err) + assert.Nil(rw.LastInitializedRound()) + assert.Equal([32]byte{}, rw.LastInitializedBlockHash()) +} + func defaultMiniHeader() *blockwatch.MiniHeader { block := &blockwatch.MiniHeader{ Number: big.NewInt(450), From 85402294544dd27740afc5602064b1909de4cbf7 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Wed, 21 Aug 2019 16:10:36 +0200 Subject: [PATCH 05/72] cmd, eth: exit when watchers error Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: e2e86f042034e50c378c5167f3bd36452b6c9da1 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 8 ++++---- eth/watchers/roundswatcher_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 20031bd..37a8651 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -64,14 +64,14 @@ func (rw *RoundsWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) } // Watch the blockwatch subscription for NewRound events -func (rw *RoundsWatcher) Watch() { +func (rw *RoundsWatcher) Watch() error { lr, err := rw.lpEth.LastInitializedRound() if err != nil { - glog.Errorf("error fetching initial lastInitializedRound value: %v", err) + return fmt.Errorf("error fetching initial lastInitializedRound value: %v", err) } bh, err := rw.lpEth.BlockHashForRound(lr) if err != nil { - glog.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) + return fmt.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) } rw.setLastInitializedRound(lr, bh) @@ -81,7 +81,7 @@ func (rw *RoundsWatcher) Watch() { for { select { case <-rw.quit: - return + return nil case err := <-sub.Err(): glog.Error(err) case events := <-events: diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go index 9ead20c..b6e0a3c 100644 --- a/eth/watchers/roundswatcher_test.go +++ b/eth/watchers/roundswatcher_test.go @@ -1,6 +1,7 @@ package watchers import ( + "errors" "math/big" "testing" "time" @@ -82,6 +83,16 @@ func TestWatchAndStop(t *testing.T) { rw.Stop() time.Sleep(2 * time.Millisecond) assert.True(watcher.sub.unsubscribed) + + // Test watch error when RPC calls fail + rw = &RoundsWatcher{ + lpEth: ð.StubClient{ + RoundsErr: errors.New("roundswatcher error"), + }, + } + err = rw.Watch() + assert.NotNil(err) + assert.Contains(err.Error(), "roundswatcher error") } func TestRoundsWatcher_HandleLog(t *testing.T) { From e20213ba6b313c0c7846e5c748e2e3792d9d69ce Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Wed, 21 Aug 2019 02:42:39 +0200 Subject: [PATCH 06/72] cmd: init senderwatcher on pm.Sender Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: a0465f78ade5bc4ccbb6afda847d145df6d3dc04 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher_test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go index b6e0a3c..7af30ee 100644 --- a/eth/watchers/roundswatcher_test.go +++ b/eth/watchers/roundswatcher_test.go @@ -7,10 +7,8 @@ import ( "time" ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/livepeer/go-livepeer/eth" "github.com/livepeer/go-livepeer/eth/blockwatch" - "github.com/livepeer/go-livepeer/pm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -112,18 +110,3 @@ func TestRoundsWatcher_HandleLog(t *testing.T) { assert.Nil(rw.LastInitializedRound()) assert.Equal([32]byte{}, rw.LastInitializedBlockHash()) } - -func defaultMiniHeader() *blockwatch.MiniHeader { - block := &blockwatch.MiniHeader{ - Number: big.NewInt(450), - Parent: pm.RandHash(), - Hash: pm.RandHash(), - } - log := types.Log{ - Topics: []ethcommon.Hash{pm.RandHash(), pm.RandHash()}, - Data: pm.RandBytes(32), - BlockHash: block.Hash, - } - block.Logs = []types.Log{log} - return block -} From 6a222786609fe2ab45927c2085e2eddefcd1a608 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 29 Aug 2019 19:23:43 +0200 Subject: [PATCH 07/72] eth: add transcoderPoolSize to RoundsWatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 5cc2e35221b1fbf1d994d17d6e6a0a128b010d81 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 32 ++++++++++++++++++++++++++++++ eth/watchers/roundswatcher_test.go | 21 ++++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 37a8651..1c24b9c 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -19,6 +19,7 @@ type RoundsWatcher struct { mu sync.RWMutex lastInitializedRound *big.Int lastInitializedBlockHash [32]byte + transcoderPoolSize *big.Int quit chan struct{} @@ -63,6 +64,18 @@ func (rw *RoundsWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) rw.lastInitializedBlockHash = hash } +func (rw *RoundsWatcher) GetTranscoderPoolSize() *big.Int { + rw.mu.RLock() + defer rw.mu.RUnlock() + return rw.transcoderPoolSize +} + +func (rw *RoundsWatcher) setTranscoderPoolSize(size *big.Int) { + rw.mu.Lock() + defer rw.mu.Unlock() + rw.transcoderPoolSize = size +} + // Watch the blockwatch subscription for NewRound events func (rw *RoundsWatcher) Watch() error { lr, err := rw.lpEth.LastInitializedRound() @@ -75,6 +88,10 @@ func (rw *RoundsWatcher) Watch() error { } rw.setLastInitializedRound(lr, bh) + if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { + glog.Errorf("error fetching initial transcoderPoolSize: %v", err) + } + events := make(chan []*blockwatch.Event, 10) sub := rw.watcher.Subscribe(events) defer sub.Unsubscribe() @@ -136,5 +153,20 @@ func (rw *RoundsWatcher) handleLog(log types.Log) error { } else { rw.setLastInitializedRound(nr.Round, nr.BlockHash) } + + // Get the active transcoder pool size when we receive a NewRound event + if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { + return err + } + + return nil +} + +func (rw *RoundsWatcher) fetchAndSetTranscoderPoolSize() error { + size, err := rw.lpEth.GetTranscoderPoolSize() + if err != nil { + return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + } + rw.setTranscoderPoolSize(size) return nil } diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go index 7af30ee..663cece 100644 --- a/eth/watchers/roundswatcher_test.go +++ b/eth/watchers/roundswatcher_test.go @@ -29,9 +29,21 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { assert.Equal(h, hash) } -func TestWatchAndStop(t *testing.T) { +func TestSetAndGet_TranscoderPoolSize(t *testing.T) { assert := assert.New(t) - lpEth := ð.StubClient{} + rw := &RoundsWatcher{} + size := big.NewInt(50) + rw.setTranscoderPoolSize(size) + assert.Equal(size, rw.transcoderPoolSize) + assert.Equal(size, rw.GetTranscoderPoolSize()) +} + +func TestRoundsWatcher_WatchAndStop(t *testing.T) { + assert := assert.New(t) + size := big.NewInt(50) + lpEth := ð.StubClient{ + PoolSize: size, + } watcher := &stubBlockWatcher{} rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) @@ -48,6 +60,7 @@ func TestWatchAndStop(t *testing.T) { go rw.Watch() time.Sleep(2 * time.Millisecond) + // New Round event watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) lastRound := rw.LastInitializedRound() @@ -56,8 +69,10 @@ func TestWatchAndStop(t *testing.T) { var expectedHashForRound [32]byte copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) + assert.Equal(size, rw.GetTranscoderPoolSize()) // Test no NewRound events, values on rw remain the same + rw.setTranscoderPoolSize(big.NewInt(10)) blockEvent.BlockHeader.Logs = header.Logs[:1] watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) @@ -66,6 +81,7 @@ func TestWatchAndStop(t *testing.T) { bhForRound = rw.LastInitializedBlockHash() copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) + assert.Equal(big.NewInt(10), rw.GetTranscoderPoolSize()) // Test RPC paths (event removed) blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) @@ -76,6 +92,7 @@ func TestWatchAndStop(t *testing.T) { bhForRound = rw.LastInitializedBlockHash() assert.Equal(lastRound.Int64(), int64(0)) assert.Equal(bhForRound, [32]byte{}) + assert.Equal(size, rw.GetTranscoderPoolSize()) // Test Stop rw.Stop() From c5cd25d6b8a712bc80c124fd2e354c8cfd29e9ab Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Tue, 3 Sep 2019 14:03:11 +0200 Subject: [PATCH 08/72] pm: updates to Broker and SenderManager interface Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 5b74602591c67645a422b5414d0fb052158248e4 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 1c24b9c..61e370b 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -89,7 +89,7 @@ func (rw *RoundsWatcher) Watch() error { rw.setLastInitializedRound(lr, bh) if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { - glog.Errorf("error fetching initial transcoderPoolSize: %v", err) + return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) } events := make(chan []*blockwatch.Event, 10) From 7d3e8c03a9570392271475378381aabe0bad8a22 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Fri, 4 Oct 2019 06:01:01 +0200 Subject: [PATCH 09/72] cmd, eth: RoundsWatcher subscription for SenderWatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 30942f652de14e5ac1eea7a7ea7e0ff7c6cc7f7d Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 61e370b..d75b319 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -7,6 +7,7 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" "github.com/livepeer/go-livepeer/eth/contracts" "github.com/golang/glog" @@ -23,6 +24,8 @@ type RoundsWatcher struct { quit chan struct{} + subscription chan<- types.Log + watcher BlockWatcher lpEth eth.LivepeerEthClient dec *EventDecoder @@ -107,8 +110,19 @@ func (rw *RoundsWatcher) Watch() error { } } +// Subscribe allows one to subscribe to the round events emitted by the Watcher. +// To unsubscribe, simply call `Unsubscribe` on the returned subscription. +// The sink channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. +func (rw *RoundsWatcher) Subscribe(sink chan<- types.Log) event.Subscription { + return rw.subScope.Track(rw.subFeed.Subscribe(sink)) +} + // Stop watching for NewRound events func (rw *RoundsWatcher) Stop() { + if rw.subscription != nil { + close(rw.subscription) + } close(rw.quit) } @@ -140,6 +154,9 @@ func (rw *RoundsWatcher) handleLog(log types.Log) error { if err := rw.dec.Decode("NewRound", log, &nr); err != nil { return fmt.Errorf("unable to decode event: %v", err) } + + rw.subFeed.Send(log) + if log.Removed { lr, err := rw.lpEth.LastInitializedRound() if err != nil { From 2e277805db3519fc99f17acc13aa99e2ea2cc7bd Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Sat, 5 Oct 2019 04:08:45 +0200 Subject: [PATCH 10/72] eth/watchers: refactor RoundsWatcher Subscription mechanism Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 1799b33eafcb50fa48171dd5509a50d05fb8fab4 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index d75b319..6384fad 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -24,7 +24,8 @@ type RoundsWatcher struct { quit chan struct{} - subscription chan<- types.Log + subFeed event.Feed + subScope event.SubscriptionScope // Subscription scope tracking current live listeners watcher BlockWatcher lpEth eth.LivepeerEthClient @@ -110,7 +111,7 @@ func (rw *RoundsWatcher) Watch() error { } } -// Subscribe allows one to subscribe to the round events emitted by the Watcher. +// Subscribe allows one to subscribe to the block events emitted by the Watcher. // To unsubscribe, simply call `Unsubscribe` on the returned subscription. // The sink channel should have ample buffer space to avoid blocking other subscribers. // Slow subscribers are not dropped. @@ -120,10 +121,8 @@ func (rw *RoundsWatcher) Subscribe(sink chan<- types.Log) event.Subscription { // Stop watching for NewRound events func (rw *RoundsWatcher) Stop() { - if rw.subscription != nil { - close(rw.subscription) - } close(rw.quit) + rw.subScope.Close() } func (rw *RoundsWatcher) handleBlockEvents(events []*blockwatch.Event) { From 69baa441e2db1a6f6c605f435753f96119e5da54 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 28 Oct 2019 23:49:07 +0100 Subject: [PATCH 11/72] eth/blockwatch: remove block header not found info statement Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 894b8db2cf0a98a4ed9175e6063dd0dce1114e7a Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 1 - 1 file changed, 1 deletion(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index d2a0a00..6cde647 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -164,7 +164,6 @@ func (w *Watcher) pollNextBlock() error { nextHeader, err := w.client.HeaderByNumber(nextBlockNumber) if err != nil { if err == ethereum.NotFound { - glog.Infof("block header not found blockNumber=%v", nextBlockNumber) return nil // Noop and wait next polling interval } return err From 8f8ca8bdb2dc8cf4b212aba05141c8939faaad5e Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 4 Nov 2019 11:46:50 +0100 Subject: [PATCH 12/72] eth/blockwatch: BackfillStartBlock option Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 74fc8f4cf6996533392f36b2ab17df320a958849 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 42 +++++++++++----- eth/blockwatch/block_watcher_test.go | 74 ++++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 16 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 6cde647..9789ab0 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -45,6 +45,7 @@ type Config struct { Store MiniHeaderStore PollingInterval time.Duration StartBlockDepth rpc.BlockNumber + BackfillStartBlock *big.Int BlockRetentionLimit int WithLogs bool Topics []common.Hash @@ -57,6 +58,7 @@ type Config struct { type Watcher struct { blockRetentionLimit int startBlockDepth rpc.BlockNumber + backfillStartBlock *big.Int stack *Stack client Client blockFeed event.Feed @@ -77,6 +79,7 @@ func New(config Config) *Watcher { pollingInterval: config.PollingInterval, blockRetentionLimit: config.BlockRetentionLimit, startBlockDepth: config.StartBlockDepth, + backfillStartBlock: config.BackfillStartBlock, stack: stack, client: config.Client, withLogs: config.WithLogs, @@ -281,28 +284,43 @@ func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, error) { events := []*Event{} + var ( + startBlockNum int + latestRetainedBlockNum int + blocksElapsed int + ) + latestRetainedBlock, err := w.stack.Peek() if err != nil { return events, err } - // No blocks stored, nowhere to backfill to - if latestRetainedBlock == nil { - return events, nil - } + latestBlock, err := w.client.HeaderByNumber(nil) if err != nil { return events, err } - blocksElapsed := big.NewInt(0).Sub(latestBlock.Number, latestRetainedBlock.Number) - if blocksElapsed.Int64() == 0 { + latestBlockNum := int(latestBlock.Number.Int64()) + + if latestRetainedBlock != nil { + latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) + // Events for latestRetainedBlock already processed, start at latestRetainedBlock + 1 + startBlockNum = latestRetainedBlockNum + 1 + } else if w.backfillStartBlock != nil { + startBlockNum = int(w.backfillStartBlock.Int64()) + } else { return events, nil } - glog.Infof("Some blocks have elapsed since last boot. Backfilling block events (this can take a while)... blocksElapsed=%v", blocksElapsed.Int64()) - startBlockNum := int(latestRetainedBlock.Number.Int64() + 1) - endBlockNum := int(latestRetainedBlock.Number.Int64() + blocksElapsed.Int64()) + if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed == 0 { + return events, nil + } + + glog.Infof("Backfilling block events (this can take a while)...\n") + glog.Infof("Start block: %v End block: %v Blocks elapsed: %v\n", startBlockNum, startBlockNum+blocksElapsed, blocksElapsed) + + endBlockNum := startBlockNum + blocksElapsed logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, endBlockNum) - if int64(furthestBlockProcessed) > latestRetainedBlock.Number.Int64() { + if furthestBlockProcessed > latestRetainedBlockNum { // If we have processed blocks further then the latestRetainedBlock in the DB, we // want to remove all blocks from the DB and insert the furthestBlockProcessed // Doing so will cause the BlockWatcher to start from that furthestBlockProcessed. @@ -337,9 +355,9 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro for _, log := range logs { blockHeader, ok := hashToBlockHeader[log.BlockHash] if !ok { - // TODO(fabio): Find a way to include the parent hash for the block as well. + // TODO: Find a way to include the parent hash for the block as well. // It's currently not an issue to omit it since we don't use the parent hash - // when processing block events in OrderWatcher. + // when processing block events in event watcher services blockHeader = &MiniHeader{ Hash: log.BlockHash, Number: big.NewInt(0).SetUint64(log.BlockNumber), diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 4f11787..06e4311 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -214,6 +214,48 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { assert.Equal(t, big.NewInt(30), headers[0].Number) } +func TestGetMissedEventsToBackfill_BackfillStartBlock(t *testing.T) { + // Fixture will return block 30 as the tip of the chain + fakeClient, err := newFakeClient("testdata/fake_client_fast_sync_fixture.json") + require.NoError(t, err) + + store := &stubMiniHeaderStore{} + + config.Store = store + config.Client = fakeClient + config.BackfillStartBlock = big.NewInt(0) + watcher := New(config) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + events, err := watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 1) + + // Check that block 30 is now in the DB as it is the last processed block + headers, err := store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 1) + assert.Equal(t, big.NewInt(30), headers[0].Number) + + store = &stubMiniHeaderStore{} + config.Store = store + config.BackfillStartBlock = big.NewInt(5) + watcher = New(config) + events, err = watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 1) + + // Check that block 30 is now in the DB as it is the last processed block + headers, err = store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 1) + assert.Equal(t, big.NewInt(30), headers[0].Number) + + config.BackfillStartBlock = nil +} + func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { // Fixture will return block 5 as the tip of the chain fakeClient, err := newFakeClient("testdata/fake_client_basic_fixture.json") @@ -246,6 +288,30 @@ func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { assert.Equal(t, big.NewInt(5), headers[0].Number) } +func TestGetMissedEventsToBackfill_NOOP(t *testing.T) { + // No last retained block + // No config.BackfillStartBlock + + fakeClient, err := newFakeClient("testdata/fake_client_basic_fixture.json") + require.NoError(t, err) + + store := &stubMiniHeaderStore{} + + config.Store = store + config.Client = fakeClient + watcher := New(config) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events, err := watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 0) + + headers, err := store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 0) +} + var logStub = types.Log{ Address: common.HexToAddress("0x21ab6c9fac80c59d401b37cb43f81ea9dde7fe34"), Topics: []common.Hash{ @@ -399,7 +465,7 @@ func TestGetLogsInBlockRange(t *testing.T) { }, }, }, - Logs: []types.Log{logStub}, + Logs: []types.Log{logStub}, FurthestBlockProcessed: to, }, logsInBlockRangeTestCase{ @@ -418,7 +484,7 @@ func TestGetLogsInBlockRange(t *testing.T) { }, }, }, - Logs: []types.Log{logStub, logStub}, + Logs: []types.Log{logStub, logStub}, FurthestBlockProcessed: from + maxBlocksInGetLogsQuery + 10, }, logsInBlockRangeTestCase{ @@ -442,7 +508,7 @@ func TestGetLogsInBlockRange(t *testing.T) { }, }, }, - Logs: []types.Log{}, + Logs: []types.Log{}, FurthestBlockProcessed: from - 1, }, logsInBlockRangeTestCase{ @@ -458,7 +524,7 @@ func TestGetLogsInBlockRange(t *testing.T) { aRange(from+maxBlocksInGetLogsQuery, from+maxBlocksInGetLogsQuery+10): filterLogsResponse{ Err: errUnexpected, }}, - Logs: []types.Log{logStub}, + Logs: []types.Log{logStub}, FurthestBlockProcessed: from + maxBlocksInGetLogsQuery - 1, }, } From 93ca42afc3e43ba83689866fb379abdd926f809a Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 4 Nov 2019 12:21:50 +0100 Subject: [PATCH 13/72] cmd: determine backfilling startblock based on currentround and last seen block Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 7d21c328e49e0afc2929c7dbb8d42e31ebd6de04 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 9789ab0..430b902 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -301,12 +301,12 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro } latestBlockNum := int(latestBlock.Number.Int64()) - if latestRetainedBlock != nil { + if w.backfillStartBlock != nil { + startBlockNum = int(w.backfillStartBlock.Int64()) + } else if latestRetainedBlock != nil { latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) // Events for latestRetainedBlock already processed, start at latestRetainedBlock + 1 startBlockNum = latestRetainedBlockNum + 1 - } else if w.backfillStartBlock != nil { - startBlockNum = int(w.backfillStartBlock.Int64()) } else { return events, nil } @@ -318,8 +318,7 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro glog.Infof("Backfilling block events (this can take a while)...\n") glog.Infof("Start block: %v End block: %v Blocks elapsed: %v\n", startBlockNum, startBlockNum+blocksElapsed, blocksElapsed) - endBlockNum := startBlockNum + blocksElapsed - logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, endBlockNum) + logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, latestBlockNum) if furthestBlockProcessed > latestRetainedBlockNum { // If we have processed blocks further then the latestRetainedBlock in the DB, we // want to remove all blocks from the DB and insert the furthestBlockProcessed From 91c1b8fe77f7d2e9a72af5271960221047350c8f Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Fri, 15 Nov 2019 10:22:53 -0500 Subject: [PATCH 14/72] blockwatch: Log Watcher error Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 2fb674c587760f121695bffb9415fa99b89ec85d Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 430b902..0412197 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -121,7 +121,7 @@ func (w *Watcher) Watch(ctx context.Context) error { return nil case <-ticker.C: if err := w.pollNextBlock(); err != nil { - glog.Error("blockwatch.Watcher error encountered") + glog.Errorf("blockwatch.Watcher error encountered: %v", err) } } } From f25291e2950ebd2828421e55841eba05ec38ed47 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Fri, 24 Jan 2020 15:29:31 +0100 Subject: [PATCH 15/72] eth/blockwatch: reduce logging level of recoverable errors and change their language to be perceived as not interfering with node operations Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: d857c1b926b38b2a749aa413e382a8abca1bc91d Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 0412197..955b6b5 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -121,7 +121,7 @@ func (w *Watcher) Watch(ctx context.Context) error { return nil case <-ticker.C: if err := w.pollNextBlock(); err != nil { - glog.Errorf("blockwatch.Watcher error encountered: %v", err) + glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%v", err) } } } @@ -199,7 +199,7 @@ func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ( // returned. This is expected to happen sometimes, and we simply return the events gathered so // far and pick back up where we left off on the next polling interval. if isUnknownBlockErr(err) { - glog.Infof("failed to get logs for block nextHeader=%v", nextHeader) + glog.V(5).Infof("missing logs for blockNumber=%v - fetching on next polling interval", nextHeader.Number) return events, nil } return events, err @@ -227,7 +227,7 @@ func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ( nextParentHeader, err := w.client.HeaderByHash(nextHeader.Parent) if err != nil { if err == ethereum.NotFound { - glog.Infof("block header not found blockHash=%v", nextHeader.Parent.Hex()) + glog.V(5).Infof("block header not found blockHash=%v - fetching on next polling interval", nextHeader.Parent.Hex()) // Noop and wait next polling interval. We remove the popped blocks // and refetch them on the next polling interval. return events, nil @@ -245,7 +245,7 @@ func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ( // returned. This is expected to happen sometimes, and we simply return the events gathered so // far and pick back up where we left off on the next polling interval. if isUnknownBlockErr(err) { - glog.Infof("failed to get logs for block nextHeader=%v", nextHeader) + glog.V(5).Infof("missing logs for blockNumber=%v - fetching on next polling interval", nextHeader.Number) return events, nil } return events, err From b9ebbcee682dd9221e586b057ddf212b46dbfff7 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 6 Feb 2020 03:31:58 +0100 Subject: [PATCH 16/72] eth/watchers: extend roundsWatcher to be a more generic timeWatcher that watchers for both latest rounds and latest blocks Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: b4605f705bcad5e355bb0c738d784e4503089900 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 188 ----------------------- eth/watchers/roundswatcher_test.go | 129 ---------------- eth/watchers/timewatcher.go | 233 +++++++++++++++++++++++++++++ eth/watchers/timewatcher_test.go | 215 ++++++++++++++++++++++++++ 4 files changed, 448 insertions(+), 317 deletions(-) delete mode 100644 eth/watchers/roundswatcher.go delete mode 100644 eth/watchers/roundswatcher_test.go create mode 100644 eth/watchers/timewatcher.go create mode 100644 eth/watchers/timewatcher_test.go diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go deleted file mode 100644 index 6384fad..0000000 --- a/eth/watchers/roundswatcher.go +++ /dev/null @@ -1,188 +0,0 @@ -package watchers - -import ( - "fmt" - "math/big" - "sync" - - ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - "github.com/livepeer/go-livepeer/eth/contracts" - - "github.com/golang/glog" - "github.com/livepeer/go-livepeer/eth" - "github.com/livepeer/go-livepeer/eth/blockwatch" -) - -// RoundsWatcher is a type for a thread safe in-memory cache that watches on-chain events for new initialized rounds -type RoundsWatcher struct { - mu sync.RWMutex - lastInitializedRound *big.Int - lastInitializedBlockHash [32]byte - transcoderPoolSize *big.Int - - quit chan struct{} - - subFeed event.Feed - subScope event.SubscriptionScope // Subscription scope tracking current live listeners - - watcher BlockWatcher - lpEth eth.LivepeerEthClient - dec *EventDecoder -} - -// NewRoundsWatcher creates a new instance of RoundsWatcher and sets the initial cache through an RPC call to an ethereum node -func NewRoundsWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, lpEth eth.LivepeerEthClient) (*RoundsWatcher, error) { - dec, err := NewEventDecoder(roundsManagerAddr, contracts.RoundsManagerABI) - if err != nil { - return nil, fmt.Errorf("error creating decoder: %v", err) - } - - return &RoundsWatcher{ - quit: make(chan struct{}), - watcher: watcher, - lpEth: lpEth, - dec: dec, - }, nil -} - -// LastInitializedRound gets the last initialized round from cache -func (rw *RoundsWatcher) LastInitializedRound() *big.Int { - rw.mu.RLock() - defer rw.mu.RUnlock() - return rw.lastInitializedRound -} - -// LastInitializedBlockHash returns the blockhash of the block the last round was initiated in -func (rw *RoundsWatcher) LastInitializedBlockHash() [32]byte { - rw.mu.RLock() - defer rw.mu.RUnlock() - return rw.lastInitializedBlockHash -} - -func (rw *RoundsWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) { - rw.mu.Lock() - defer rw.mu.Unlock() - rw.lastInitializedRound = round - rw.lastInitializedBlockHash = hash -} - -func (rw *RoundsWatcher) GetTranscoderPoolSize() *big.Int { - rw.mu.RLock() - defer rw.mu.RUnlock() - return rw.transcoderPoolSize -} - -func (rw *RoundsWatcher) setTranscoderPoolSize(size *big.Int) { - rw.mu.Lock() - defer rw.mu.Unlock() - rw.transcoderPoolSize = size -} - -// Watch the blockwatch subscription for NewRound events -func (rw *RoundsWatcher) Watch() error { - lr, err := rw.lpEth.LastInitializedRound() - if err != nil { - return fmt.Errorf("error fetching initial lastInitializedRound value: %v", err) - } - bh, err := rw.lpEth.BlockHashForRound(lr) - if err != nil { - return fmt.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) - } - rw.setLastInitializedRound(lr, bh) - - if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) - } - - events := make(chan []*blockwatch.Event, 10) - sub := rw.watcher.Subscribe(events) - defer sub.Unsubscribe() - for { - select { - case <-rw.quit: - return nil - case err := <-sub.Err(): - glog.Error(err) - case events := <-events: - rw.handleBlockEvents(events) - } - } -} - -// Subscribe allows one to subscribe to the block events emitted by the Watcher. -// To unsubscribe, simply call `Unsubscribe` on the returned subscription. -// The sink channel should have ample buffer space to avoid blocking other subscribers. -// Slow subscribers are not dropped. -func (rw *RoundsWatcher) Subscribe(sink chan<- types.Log) event.Subscription { - return rw.subScope.Track(rw.subFeed.Subscribe(sink)) -} - -// Stop watching for NewRound events -func (rw *RoundsWatcher) Stop() { - close(rw.quit) - rw.subScope.Close() -} - -func (rw *RoundsWatcher) handleBlockEvents(events []*blockwatch.Event) { - for _, event := range events { - for _, log := range event.BlockHeader.Logs { - if event.Type == blockwatch.Removed { - log.Removed = true - } - if err := rw.handleLog(log); err != nil { - glog.Error(err) - } - } - } -} - -func (rw *RoundsWatcher) handleLog(log types.Log) error { - eventName, err := rw.dec.FindEventName(log) - if err != nil { - // Noop if we cannot find the event name - return nil - } - - if eventName != "NewRound" { - return fmt.Errorf("eventName is not NewRound") - } - - var nr contracts.RoundsManagerNewRound - if err := rw.dec.Decode("NewRound", log, &nr); err != nil { - return fmt.Errorf("unable to decode event: %v", err) - } - - rw.subFeed.Send(log) - - if log.Removed { - lr, err := rw.lpEth.LastInitializedRound() - if err != nil { - return err - } - bh, err := rw.lpEth.BlockHashForRound(lr) - if err != nil { - return err - } - rw.setLastInitializedRound(lr, bh) - } else { - rw.setLastInitializedRound(nr.Round, nr.BlockHash) - } - - // Get the active transcoder pool size when we receive a NewRound event - if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { - return err - } - - return nil -} - -func (rw *RoundsWatcher) fetchAndSetTranscoderPoolSize() error { - size, err := rw.lpEth.GetTranscoderPoolSize() - if err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) - } - rw.setTranscoderPoolSize(size) - return nil -} diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go deleted file mode 100644 index 663cece..0000000 --- a/eth/watchers/roundswatcher_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package watchers - -import ( - "errors" - "math/big" - "testing" - "time" - - ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/livepeer/go-livepeer/eth" - "github.com/livepeer/go-livepeer/eth/blockwatch" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { - assert := assert.New(t) - rw := &RoundsWatcher{} - round := big.NewInt(5) - var hash [32]byte - copy(hash[:], "hello world") - rw.setLastInitializedRound(round, hash) - assert.Equal(rw.lastInitializedRound, round) - assert.Equal(rw.lastInitializedBlockHash, hash) - - r := rw.LastInitializedRound() - assert.Equal(r, round) - h := rw.LastInitializedBlockHash() - assert.Equal(h, hash) -} - -func TestSetAndGet_TranscoderPoolSize(t *testing.T) { - assert := assert.New(t) - rw := &RoundsWatcher{} - size := big.NewInt(50) - rw.setTranscoderPoolSize(size) - assert.Equal(size, rw.transcoderPoolSize) - assert.Equal(size, rw.GetTranscoderPoolSize()) -} - -func TestRoundsWatcher_WatchAndStop(t *testing.T) { - assert := assert.New(t) - size := big.NewInt(50) - lpEth := ð.StubClient{ - PoolSize: size, - } - watcher := &stubBlockWatcher{} - rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) - assert.Nil(err) - - header := defaultMiniHeader() - newRoundEvent := newStubNewRoundLog() - - header.Logs = append(header.Logs, newRoundEvent) - blockEvent := &blockwatch.Event{ - Type: blockwatch.Added, - BlockHeader: header, - } - - go rw.Watch() - time.Sleep(2 * time.Millisecond) - - // New Round event - watcher.sink <- []*blockwatch.Event{blockEvent} - time.Sleep(2 * time.Millisecond) - lastRound := rw.LastInitializedRound() - assert.Zero(lastRound.Cmp(big.NewInt(8))) - bhForRound := rw.LastInitializedBlockHash() - var expectedHashForRound [32]byte - copy(expectedHashForRound[:], newRoundEvent.Data[:]) - assert.Equal(bhForRound, expectedHashForRound) - assert.Equal(size, rw.GetTranscoderPoolSize()) - - // Test no NewRound events, values on rw remain the same - rw.setTranscoderPoolSize(big.NewInt(10)) - blockEvent.BlockHeader.Logs = header.Logs[:1] - watcher.sink <- []*blockwatch.Event{blockEvent} - time.Sleep(2 * time.Millisecond) - lastRound = rw.LastInitializedRound() - assert.Zero(lastRound.Cmp(big.NewInt(8))) - bhForRound = rw.LastInitializedBlockHash() - copy(expectedHashForRound[:], newRoundEvent.Data[:]) - assert.Equal(bhForRound, expectedHashForRound) - assert.Equal(big.NewInt(10), rw.GetTranscoderPoolSize()) - - // Test RPC paths (event removed) - blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) - blockEvent.Type = blockwatch.Removed - watcher.sink <- []*blockwatch.Event{blockEvent} - time.Sleep(2 * time.Millisecond) - lastRound = rw.LastInitializedRound() - bhForRound = rw.LastInitializedBlockHash() - assert.Equal(lastRound.Int64(), int64(0)) - assert.Equal(bhForRound, [32]byte{}) - assert.Equal(size, rw.GetTranscoderPoolSize()) - - // Test Stop - rw.Stop() - time.Sleep(2 * time.Millisecond) - assert.True(watcher.sub.unsubscribed) - - // Test watch error when RPC calls fail - rw = &RoundsWatcher{ - lpEth: ð.StubClient{ - RoundsErr: errors.New("roundswatcher error"), - }, - } - err = rw.Watch() - assert.NotNil(err) - assert.Contains(err.Error(), "roundswatcher error") -} - -func TestRoundsWatcher_HandleLog(t *testing.T) { - lpEth := ð.StubClient{} - watcher := &stubBlockWatcher{} - rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) - require.Nil(t, err) - - assert := assert.New(t) - - // Test unknown event - log := newStubBaseLog() - log.Topics = []ethcommon.Hash{ethcommon.BytesToHash([]byte("foo"))} - - err = rw.handleLog(log) - assert.Nil(err) - assert.Nil(rw.LastInitializedRound()) - assert.Equal([32]byte{}, rw.LastInitializedBlockHash()) -} diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go new file mode 100644 index 0000000..e61dd23 --- /dev/null +++ b/eth/watchers/timewatcher.go @@ -0,0 +1,233 @@ +package watchers + +import ( + "fmt" + "math/big" + "sync" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/livepeer/go-livepeer/eth/contracts" + + "github.com/golang/glog" + "github.com/livepeer/go-livepeer/eth" + "github.com/livepeer/go-livepeer/eth/blockwatch" +) + +// TimeWatcher is a type for a thread safe in-memory cache that watches for the following on-chain events: +// * New rounds +// * New blocks +// * Transcoder (de)activation + +// TimeWatcher allows for subscriptions to certain data feeds using a caller provided sink channel +// consumers of the TimeWatcher can subscribe to following data feeds: +// * Last Initialized Round Number +// * Last Seen Block Number +type TimeWatcher struct { + // state + mu sync.RWMutex + lastInitializedRound *big.Int + lastInitializedBlockHash [32]byte + transcoderPoolSize *big.Int + lastSeenBlock *big.Int + + // last initialized round number subscription feeds + roundSubFeed event.Feed + roundSubScope event.SubscriptionScope + // last seen block number subscription feeds + blockSubFeed event.Feed + blockSubScope event.SubscriptionScope + + watcher BlockWatcher + lpEth eth.LivepeerEthClient + dec *EventDecoder + + quit chan struct{} +} + +// NewTimeWatcher creates a new instance of TimeWatcher and sets the initial cache through an RPC call to an ethereum node +func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, lpEth eth.LivepeerEthClient) (*TimeWatcher, error) { + dec, err := NewEventDecoder(roundsManagerAddr, contracts.RoundsManagerABI) + if err != nil { + return nil, fmt.Errorf("error creating decoder: %v", err) + } + + return &TimeWatcher{ + quit: make(chan struct{}), + watcher: watcher, + lpEth: lpEth, + dec: dec, + }, nil +} + +// LastInitializedRound gets the last initialized round from cache +func (tw *TimeWatcher) LastInitializedRound() *big.Int { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.lastInitializedRound +} + +// LastInitializedBlockHash returns the blockhash of the block the last round was initiated in +func (tw *TimeWatcher) LastInitializedBlockHash() [32]byte { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.lastInitializedBlockHash +} + +func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) { + tw.mu.Lock() + defer tw.mu.Unlock() + tw.lastInitializedRound = round + tw.lastInitializedBlockHash = hash +} + +func (tw *TimeWatcher) GetTranscoderPoolSize() *big.Int { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.transcoderPoolSize +} + +func (tw *TimeWatcher) setTranscoderPoolSize(size *big.Int) { + tw.mu.Lock() + defer tw.mu.Unlock() + tw.transcoderPoolSize = size +} + +func (tw *TimeWatcher) LastSeenBlock() *big.Int { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.lastSeenBlock +} + +func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { + tw.mu.Lock() + defer tw.mu.Unlock() + tw.lastSeenBlock = blockNum +} + +// Watch the blockwatch subscription for NewRound events +func (tw *TimeWatcher) Watch() error { + lr, err := tw.lpEth.LastInitializedRound() + if err != nil { + return fmt.Errorf("error fetching initial lastInitializedRound value: %v", err) + } + bh, err := tw.lpEth.BlockHashForRound(lr) + if err != nil { + return fmt.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) + } + tw.setLastInitializedRound(lr, bh) + + if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { + return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + } + + events := make(chan []*blockwatch.Event, 10) + sub := tw.watcher.Subscribe(events) + defer sub.Unsubscribe() + for { + select { + case <-tw.quit: + return nil + case err := <-sub.Err(): + glog.Error(err) + case events := <-events: + tw.handleBlockEvents(events) + } + } +} + +// SubscribeRounds allows one to subscribe to new round events +// To unsubscribe, simply call `Unsubscribe` on the returned subscription. +// The sink channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. +func (tw *TimeWatcher) SubscribeRounds(sink chan<- types.Log) event.Subscription { + return tw.roundSubScope.Track(tw.roundSubFeed.Subscribe(sink)) +} + +// SubscribeBlocks allows one to subscribe to newly seen block numbers +// To unsubscribe, simply call `Unsubscribe` on the returned subscription. +// The sink channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. +func (tw *TimeWatcher) SubscribeBlocks(sink chan<- *big.Int) event.Subscription { + return tw.blockSubScope.Track(tw.blockSubFeed.Subscribe(sink)) +} + +// Stop TimeWatcher +func (tw *TimeWatcher) Stop() { + close(tw.quit) + tw.blockSubScope.Close() + tw.roundSubScope.Close() +} + +func (tw *TimeWatcher) handleBlockEvents(events []*blockwatch.Event) { + for _, event := range events { + tw.handleBlockNum(event) + for _, log := range event.BlockHeader.Logs { + if event.Type == blockwatch.Removed { + log.Removed = true + } + if err := tw.handleLog(log); err != nil { + glog.Error(err) + } + } + } +} + +func (tw *TimeWatcher) handleBlockNum(event *blockwatch.Event) { + last := tw.LastSeenBlock() + new := event.BlockHeader.Number + if last == nil || last.Cmp(new) != 0 { + tw.setLastSeenBlock(new) + tw.blockSubFeed.Send(new) + } +} + +func (tw *TimeWatcher) handleLog(log types.Log) error { + eventName, err := tw.dec.FindEventName(log) + if err != nil { + // Noop if we cannot find the event name + return nil + } + + if eventName != "NewRound" { + return fmt.Errorf("eventName is not NewRound") + } + + var nr contracts.RoundsManagerNewRound + if err := tw.dec.Decode("NewRound", log, &nr); err != nil { + return fmt.Errorf("unable to decode event: %v", err) + } + + tw.roundSubFeed.Send(log) + + if log.Removed { + lr, err := tw.lpEth.LastInitializedRound() + if err != nil { + return err + } + bh, err := tw.lpEth.BlockHashForRound(lr) + if err != nil { + return err + } + tw.setLastInitializedRound(lr, bh) + } else { + tw.setLastInitializedRound(nr.Round, nr.BlockHash) + } + + // Get the active transcoder pool size when we receive a NewRound event + if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { + return err + } + + return nil +} + +func (tw *TimeWatcher) fetchAndSetTranscoderPoolSize() error { + size, err := tw.lpEth.GetTranscoderPoolSize() + if err != nil { + return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + } + tw.setTranscoderPoolSize(size) + return nil +} diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go new file mode 100644 index 0000000..4025dce --- /dev/null +++ b/eth/watchers/timewatcher_test.go @@ -0,0 +1,215 @@ +package watchers + +import ( + "errors" + "math/big" + "testing" + "time" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/go-livepeer/eth" + "github.com/livepeer/go-livepeer/eth/blockwatch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { + assert := assert.New(t) + tw := &TimeWatcher{} + round := big.NewInt(5) + var hash [32]byte + copy(hash[:], "hello world") + tw.setLastInitializedRound(round, hash) + assert.Equal(tw.lastInitializedRound, round) + assert.Equal(tw.lastInitializedBlockHash, hash) + + r := tw.LastInitializedRound() + assert.Equal(r, round) + h := tw.LastInitializedBlockHash() + assert.Equal(h, hash) +} + +func TestSetAndGet_TranscoderPoolSize(t *testing.T) { + assert := assert.New(t) + tw := &TimeWatcher{} + size := big.NewInt(50) + tw.setTranscoderPoolSize(size) + assert.Equal(size, tw.transcoderPoolSize) + assert.Equal(size, tw.GetTranscoderPoolSize()) +} + +func TestRoundsWatcher_WatchAndStop(t *testing.T) { + assert := assert.New(t) + size := big.NewInt(50) + lpEth := ð.StubClient{ + PoolSize: size, + } + watcher := &stubBlockWatcher{} + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(err) + + header := defaultMiniHeader() + newRoundEvent := newStubNewRoundLog() + + header.Logs = append(header.Logs, newRoundEvent) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + go tw.Watch() + time.Sleep(2 * time.Millisecond) + + // New Round event + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound := tw.LastInitializedRound() + assert.Zero(lastRound.Cmp(big.NewInt(8))) + bhForRound := tw.LastInitializedBlockHash() + var expectedHashForRound [32]byte + copy(expectedHashForRound[:], newRoundEvent.Data[:]) + assert.Equal(bhForRound, expectedHashForRound) + assert.Equal(size, tw.GetTranscoderPoolSize()) + + // Test no NewRound events, values on rw remain the same + tw.setTranscoderPoolSize(big.NewInt(10)) + blockEvent.BlockHeader.Logs = header.Logs[:1] + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound = tw.LastInitializedRound() + assert.Zero(lastRound.Cmp(big.NewInt(8))) + bhForRound = tw.LastInitializedBlockHash() + copy(expectedHashForRound[:], newRoundEvent.Data[:]) + assert.Equal(bhForRound, expectedHashForRound) + assert.Equal(big.NewInt(10), tw.GetTranscoderPoolSize()) + + // Test RPC paths (event removed) + blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) + blockEvent.Type = blockwatch.Removed + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound = tw.LastInitializedRound() + bhForRound = tw.LastInitializedBlockHash() + assert.Equal(lastRound.Int64(), int64(0)) + assert.Equal(bhForRound, [32]byte{}) + assert.Equal(size, tw.GetTranscoderPoolSize()) + + // Test Stop + tw.Stop() + time.Sleep(2 * time.Millisecond) + assert.True(watcher.sub.unsubscribed) + + // Test watch error when RPC calls fail + tw = &TimeWatcher{ + lpEth: ð.StubClient{ + RoundsErr: errors.New("roundswatcher error"), + }, + } + err = tw.Watch() + assert.NotNil(err) + assert.Contains(err.Error(), "roundswatcher error") +} + +func TestRoundsWatcher_HandleLog(t *testing.T) { + lpEth := ð.StubClient{} + watcher := &stubBlockWatcher{} + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + require.Nil(t, err) + + assert := assert.New(t) + + // Test unknown event + log := newStubBaseLog() + log.Topics = []ethcommon.Hash{ethcommon.BytesToHash([]byte("foo"))} + + err = tw.handleLog(log) + assert.Nil(err) + assert.Nil(tw.LastInitializedRound()) + assert.Equal([32]byte{}, tw.LastInitializedBlockHash()) +} + +func TestLastSeenBlock(t *testing.T) { + assert := assert.New(t) + tw := &TimeWatcher{} + block := big.NewInt(5) + + tw.setLastSeenBlock(block) + tw.LastSeenBlock() + assert.Equal(big.NewInt(5), tw.LastSeenBlock()) +} + +func TestHandleBlockNum(t *testing.T) { + assert := assert.New(t) + watcher := &stubBlockWatcher{} + + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + assert.Nil(err) + header := defaultMiniHeader() + header.Number = big.NewInt(10) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + go tw.Watch() + time.Sleep(2 * time.Millisecond) + + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + assert.Equal(tw.LastSeenBlock(), header.Number) +} + +func TestSubscribeBlocks(t *testing.T) { + assert := assert.New(t) + watcher := &stubBlockWatcher{} + + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + assert.Nil(err) + header := defaultMiniHeader() + header.Number = big.NewInt(10) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + events := make(chan *big.Int, 10) + sub := tw.SubscribeBlocks(events) + defer sub.Unsubscribe() + + go tw.Watch() + time.Sleep(2 * time.Millisecond) + + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + update := <-events + assert.Equal(update, header.Number) +} + +func TestSubscribeRounds(t *testing.T) { + assert := assert.New(t) + watcher := &stubBlockWatcher{} + + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + assert.Nil(err) + header := defaultMiniHeader() + newRoundEvent := newStubNewRoundLog() + + header.Logs = append(header.Logs, newRoundEvent) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + events := make(chan types.Log, 10) + sub := tw.SubscribeRounds(events) + defer sub.Unsubscribe() + + go tw.Watch() + time.Sleep(2 * time.Millisecond) + + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + update := <-events + assert.Equal(newRoundEvent, update) +} From bb91a19ba97064e8bf474e0c9a99ffa24be3d1e8 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Tue, 25 Feb 2020 18:33:35 +0100 Subject: [PATCH 17/72] eth/watcher: cache last seen block when starting timeWatcher Watch() loop Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 466185d0f90a1ed6179db8d9ea25834a5f9ec0a5 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index e61dd23..002ea37 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -110,16 +110,16 @@ func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { func (tw *TimeWatcher) Watch() error { lr, err := tw.lpEth.LastInitializedRound() if err != nil { - return fmt.Errorf("error fetching initial lastInitializedRound value: %v", err) + return fmt.Errorf("error fetching initial lastInitializedRound value err=%v", err) } bh, err := tw.lpEth.BlockHashForRound(lr) if err != nil { - return fmt.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) + return fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) } tw.setLastInitializedRound(lr, bh) if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + return fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) } events := make(chan []*blockwatch.Event, 10) From a5ac4c1b6ca448e3663b8a9ec2a9402ad2d7931c Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 16 Mar 2020 18:02:09 +0100 Subject: [PATCH 18/72] eth: prefill last seen block state in timewatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 06cdd9176cb4854d4134d9b424647556fa3b4045 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 10 ++++++ eth/watchers/timewatcher_test.go | 61 +++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 002ea37..10f4a38 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -122,6 +122,16 @@ func (tw *TimeWatcher) Watch() error { return fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) } + lastSeenBlock, err := tw.watcher.GetLatestBlock() + if err != nil { + return fmt.Errorf("error fetching last seen block err=%v", err) + } + blockNum := big.NewInt(0) + if lastSeenBlock != nil { + blockNum = lastSeenBlock.Number + } + tw.setLastSeenBlock(blockNum) + events := make(chan []*blockwatch.Event, 10) sub := tw.watcher.Subscribe(events) defer sub.Unsubscribe() diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index 4025dce..ea4a2cc 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/livepeer/go-livepeer/eth" @@ -39,13 +40,21 @@ func TestSetAndGet_TranscoderPoolSize(t *testing.T) { assert.Equal(size, tw.GetTranscoderPoolSize()) } -func TestRoundsWatcher_WatchAndStop(t *testing.T) { +func TestTimeWatcher_WatchAndStop(t *testing.T) { assert := assert.New(t) size := big.NewInt(50) + block := big.NewInt(10) + round := big.NewInt(1) + hash := ethcommon.HexToHash("foo") lpEth := ð.StubClient{ - PoolSize: size, + PoolSize: size, + BlockNum: block, + BlockHashToReturn: hash, + Round: round, + } + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: block}, } - watcher := &stubBlockWatcher{} tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) @@ -60,6 +69,26 @@ func TestRoundsWatcher_WatchAndStop(t *testing.T) { go tw.Watch() time.Sleep(2 * time.Millisecond) + // Check state initialization + bh := tw.LastInitializedBlockHash() + assert.Equal(hash, common.BytesToHash(bh[:])) + assert.Equal(round, tw.LastInitializedRound()) + assert.Equal(size, tw.GetTranscoderPoolSize()) + assert.Equal(block, tw.LastSeenBlock()) + tw.Stop() + + // if watcher.GetLatestBlock() == nil, initialise lastSeenBlock to big.NewInt(0) + watcher.latestHeader = nil + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(err) + go tw.Watch() + time.Sleep(2 * time.Millisecond) + // Check state initialization + bh = tw.LastInitializedBlockHash() + assert.Equal(hash, common.BytesToHash(bh[:])) + assert.Equal(round, tw.LastInitializedRound()) + assert.Equal(size, tw.GetTranscoderPoolSize()) + assert.Equal(big.NewInt(0), tw.LastSeenBlock()) // New Round event watcher.sink <- []*blockwatch.Event{blockEvent} @@ -71,6 +100,7 @@ func TestRoundsWatcher_WatchAndStop(t *testing.T) { copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) assert.Equal(size, tw.GetTranscoderPoolSize()) + assert.Equal(header.Number, tw.LastSeenBlock()) // Test no NewRound events, values on rw remain the same tw.setTranscoderPoolSize(big.NewInt(10)) @@ -83,17 +113,18 @@ func TestRoundsWatcher_WatchAndStop(t *testing.T) { copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) assert.Equal(big.NewInt(10), tw.GetTranscoderPoolSize()) + assert.Equal(header.Number, tw.LastSeenBlock()) // Test RPC paths (event removed) blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) blockEvent.Type = blockwatch.Removed watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) - lastRound = tw.LastInitializedRound() bhForRound = tw.LastInitializedBlockHash() - assert.Equal(lastRound.Int64(), int64(0)) - assert.Equal(bhForRound, [32]byte{}) + assert.Equal(hash, common.BytesToHash(bh[:])) + assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) + assert.Equal(header.Number, tw.LastSeenBlock()) // Test Stop tw.Stop() @@ -103,15 +134,15 @@ func TestRoundsWatcher_WatchAndStop(t *testing.T) { // Test watch error when RPC calls fail tw = &TimeWatcher{ lpEth: ð.StubClient{ - RoundsErr: errors.New("roundswatcher error"), + RoundsErr: errors.New("timewatcher error"), }, } err = tw.Watch() assert.NotNil(err) - assert.Contains(err.Error(), "roundswatcher error") + assert.Contains(err.Error(), "timewatcher error") } -func TestRoundsWatcher_HandleLog(t *testing.T) { +func TestTimeWatcher_HandleLog(t *testing.T) { lpEth := ð.StubClient{} watcher := &stubBlockWatcher{} tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) @@ -141,7 +172,9 @@ func TestLastSeenBlock(t *testing.T) { func TestHandleBlockNum(t *testing.T) { assert := assert.New(t) - watcher := &stubBlockWatcher{} + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) @@ -162,7 +195,9 @@ func TestHandleBlockNum(t *testing.T) { func TestSubscribeBlocks(t *testing.T) { assert := assert.New(t) - watcher := &stubBlockWatcher{} + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) @@ -188,7 +223,9 @@ func TestSubscribeBlocks(t *testing.T) { func TestSubscribeRounds(t *testing.T) { assert := assert.New(t) - watcher := &stubBlockWatcher{} + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) From 08cd77414f73eb12c9ebb6baf975fbcf7b3a43d9 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 10 Dec 2020 22:26:37 +0100 Subject: [PATCH 19/72] eth/watchers: add CurrentRoundStartBlock to TimeWatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 1b5f53fe907088672522c72a5a1d76634088bc30 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 28 ++++++++++++++++++++++------ eth/watchers/timewatcher_test.go | 4 +++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 10f4a38..138961e 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -29,6 +29,7 @@ type TimeWatcher struct { mu sync.RWMutex lastInitializedRound *big.Int lastInitializedBlockHash [32]byte + currentRoundStartBlock *big.Int transcoderPoolSize *big.Int lastSeenBlock *big.Int @@ -75,11 +76,18 @@ func (tw *TimeWatcher) LastInitializedBlockHash() [32]byte { return tw.lastInitializedBlockHash } -func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) { +func (tw *TimeWatcher) CurrentRoundStartBlock() *big.Int { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.currentRoundStartBlock +} + +func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, startBlk *big.Int) { tw.mu.Lock() defer tw.mu.Unlock() tw.lastInitializedRound = round tw.lastInitializedBlockHash = hash + tw.currentRoundStartBlock = startBlk } func (tw *TimeWatcher) GetTranscoderPoolSize() *big.Int { @@ -116,7 +124,11 @@ func (tw *TimeWatcher) Watch() error { if err != nil { return fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) } - tw.setLastInitializedRound(lr, bh) + num, err := tw.lpEth.CurrentRoundStartBlock() + if err != nil { + return fmt.Errorf("error fetching current round start block %v=", err) + } + tw.setLastInitializedRound(lr, bh, num) if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { return fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) @@ -209,8 +221,10 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return fmt.Errorf("unable to decode event: %v", err) } - tw.roundSubFeed.Send(log) - + roundStartBlock, err := tw.lpEth.CurrentRoundStartBlock() + if err != nil { + return err + } if log.Removed { lr, err := tw.lpEth.LastInitializedRound() if err != nil { @@ -220,9 +234,9 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { if err != nil { return err } - tw.setLastInitializedRound(lr, bh) + tw.setLastInitializedRound(lr, bh, roundStartBlock) } else { - tw.setLastInitializedRound(nr.Round, nr.BlockHash) + tw.setLastInitializedRound(nr.Round, nr.BlockHash, roundStartBlock) } // Get the active transcoder pool size when we receive a NewRound event @@ -230,6 +244,8 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return err } + tw.roundSubFeed.Send(log) + return nil } diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index ea4a2cc..f5c5f8d 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -21,7 +21,8 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { round := big.NewInt(5) var hash [32]byte copy(hash[:], "hello world") - tw.setLastInitializedRound(round, hash) + num := big.NewInt(10) + tw.setLastInitializedRound(round, hash, num) assert.Equal(tw.lastInitializedRound, round) assert.Equal(tw.lastInitializedBlockHash, hash) @@ -29,6 +30,7 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { assert.Equal(r, round) h := tw.LastInitializedBlockHash() assert.Equal(h, hash) + assert.Equal(tw.CurrentRoundStartBlock(), num) } func TestSetAndGet_TranscoderPoolSize(t *testing.T) { From 5d7aacecc6fe9033e7dfa52384e5318516cfa5c7 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 7 Jan 2021 00:00:18 +0100 Subject: [PATCH 20/72] eth: blockwatcher can process multiple blocks in a polling interval Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 003029a6cfba61cef7b4222941287fecdc8084e3 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 955b6b5..4f7d241 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -68,7 +68,7 @@ type Watcher struct { ticker *time.Ticker withLogs bool topics []common.Hash - mu sync.RWMutex + sync.RWMutex } // New creates a new Watcher instance. @@ -105,13 +105,13 @@ func (w *Watcher) BackfillEventsIfNeeded(ctx context.Context) error { // Watch starts the Watcher. It will continuously look for new blocks and blocks // until the given context is canceled. Typically, you want to call Watch inside a goroutine. func (w *Watcher) Watch(ctx context.Context) error { - w.mu.Lock() + w.Lock() if w.wasStartedOnce { - w.mu.Unlock() + w.Unlock() return errors.New("Can only start Watcher once per instance") } w.wasStartedOnce = true - w.mu.Unlock() + w.Unlock() ticker := time.NewTicker(w.pollingInterval) for { @@ -120,7 +120,7 @@ func (w *Watcher) Watch(ctx context.Context) error { ticker.Stop() return nil case <-ticker.C: - if err := w.pollNextBlock(); err != nil { + if err := w.syncToLatestBlock(); err != nil { glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%v", err) } } @@ -146,6 +146,27 @@ func (w *Watcher) InspectRetainedBlocks() ([]*MiniHeader, error) { return w.stack.Inspect() } +func (w *Watcher) syncToLatestBlock() error { + w.Lock() + defer w.Unlock() + newestHeader, err := w.client.HeaderByNumber(nil) + if err != nil { + return err + } + + lastSeenHeader, err := w.stack.Peek() + if err != nil { + return err + } + + for i := lastSeenHeader.Number; i.Cmp(newestHeader.Number) < 0; i = i.Add(i, big.NewInt(1)) { + if err := w.pollNextBlock(); err != nil { + return err + } + } + return nil +} + // pollNextBlock polls for the next block header to be added to the block stack. // If there are no blocks on the stack, it fetches the first block at the specified // `startBlockDepth` supplied at instantiation. From 56fea93af5552a2a054147362585e150431918df Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Tue, 6 Apr 2021 17:32:33 +0200 Subject: [PATCH 21/72] blockwatch: don't log backfilling statement when no blocks have elapsed Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 55b14ac81132831fb85dabfe4493cc72a82aa7c6 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 4f7d241..87b7d56 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -159,6 +159,10 @@ func (w *Watcher) syncToLatestBlock() error { return err } + if lastSeenHeader == nil { + return w.pollNextBlock() + } + for i := lastSeenHeader.Number; i.Cmp(newestHeader.Number) < 0; i = i.Add(i, big.NewInt(1)) { if err := w.pollNextBlock(); err != nil { return err @@ -332,7 +336,7 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro return events, nil } - if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed == 0 { + if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed <= 0 { return events, nil } From db210d18f6044c18ac30bcec6f803c8a92d72052 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 12 Apr 2021 17:05:40 +0200 Subject: [PATCH 22/72] eth: move timewatcher state initialisation into constructor to be synchronous Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 3aec3aa1628af8307315bd80123632714f9a4044 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 67 ++++++++++--------- eth/watchers/timewatcher_test.go | 110 +++++++++++++++++++++++-------- 2 files changed, 118 insertions(+), 59 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 138961e..c9fc6ca 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -54,12 +54,42 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l return nil, fmt.Errorf("error creating decoder: %v", err) } - return &TimeWatcher{ + tw := &TimeWatcher{ quit: make(chan struct{}), watcher: watcher, lpEth: lpEth, dec: dec, - }, nil + } + + lr, err := tw.lpEth.LastInitializedRound() + if err != nil { + return nil, fmt.Errorf("error fetching initial lastInitializedRound value err=%v", err) + } + bh, err := tw.lpEth.BlockHashForRound(lr) + if err != nil { + return nil, fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) + } + num, err := tw.lpEth.CurrentRoundStartBlock() + if err != nil { + return nil, fmt.Errorf("error fetching current round start block %v=", err) + } + tw.setLastInitializedRound(lr, bh, num) + + lastSeenBlock, err := tw.watcher.GetLatestBlock() + if err != nil { + return nil, fmt.Errorf("error fetching last seen block err=%v", err) + } + blockNum := big.NewInt(0) + if lastSeenBlock != nil { + blockNum = lastSeenBlock.Number + } + tw.setLastSeenBlock(blockNum) + + if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { + return nil, fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) + } + + return tw, nil } // LastInitializedRound gets the last initialized round from cache @@ -93,6 +123,9 @@ func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, st func (tw *TimeWatcher) GetTranscoderPoolSize() *big.Int { tw.mu.RLock() defer tw.mu.RUnlock() + if tw.transcoderPoolSize == nil { + return big.NewInt(0) + } return tw.transcoderPoolSize } @@ -116,34 +149,6 @@ func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { // Watch the blockwatch subscription for NewRound events func (tw *TimeWatcher) Watch() error { - lr, err := tw.lpEth.LastInitializedRound() - if err != nil { - return fmt.Errorf("error fetching initial lastInitializedRound value err=%v", err) - } - bh, err := tw.lpEth.BlockHashForRound(lr) - if err != nil { - return fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) - } - num, err := tw.lpEth.CurrentRoundStartBlock() - if err != nil { - return fmt.Errorf("error fetching current round start block %v=", err) - } - tw.setLastInitializedRound(lr, bh, num) - - if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) - } - - lastSeenBlock, err := tw.watcher.GetLatestBlock() - if err != nil { - return fmt.Errorf("error fetching last seen block err=%v", err) - } - blockNum := big.NewInt(0) - if lastSeenBlock != nil { - blockNum = lastSeenBlock.Number - } - tw.setLastSeenBlock(blockNum) - events := make(chan []*blockwatch.Event, 10) sub := tw.watcher.Subscribe(events) defer sub.Unsubscribe() @@ -252,7 +257,7 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { func (tw *TimeWatcher) fetchAndSetTranscoderPoolSize() error { size, err := tw.lpEth.GetTranscoderPoolSize() if err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + return err } tw.setTranscoderPoolSize(size) return nil diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index f5c5f8d..72da00d 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -1,7 +1,7 @@ package watchers import ( - "errors" + "fmt" "math/big" "testing" "time" @@ -40,9 +40,14 @@ func TestSetAndGet_TranscoderPoolSize(t *testing.T) { tw.setTranscoderPoolSize(size) assert.Equal(size, tw.transcoderPoolSize) assert.Equal(size, tw.GetTranscoderPoolSize()) + + // return big.Int(0) when nil + tw.setTranscoderPoolSize(nil) + assert.Nil(tw.transcoderPoolSize) + assert.Equal(big.NewInt(0), tw.GetTranscoderPoolSize()) } -func TestTimeWatcher_WatchAndStop(t *testing.T) { +func TestTimeWatcher_NewTimeWatcher(t *testing.T) { assert := assert.New(t) size := big.NewInt(50) block := big.NewInt(10) @@ -53,44 +58,103 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { BlockNum: block, BlockHashToReturn: hash, Round: round, + Errors: make(map[string]error), } watcher := &stubBlockWatcher{ latestHeader: &blockwatch.MiniHeader{Number: block}, } + + testErr := fmt.Errorf("error") + + // Last InitializedRound error + lpEth.Errors["LastInitializedRound"] = testErr + expErr := fmt.Sprintf("error fetching initial lastInitializedRound value err=%v", testErr) tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) - assert.Nil(err) + assert.Nil(tw) + assert.EqualError(err, expErr) + lpEth.Errors["LastInitializedRound"] = nil - header := defaultMiniHeader() - newRoundEvent := newStubNewRoundLog() + // BlockHashForRound error + lpEth.Errors["BlockHashForRound"] = testErr + expErr = fmt.Sprintf("error fetching initial lastInitializedBlockHash value err=%v", testErr) + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(tw) + assert.EqualError(err, expErr) + lpEth.Errors["BlockHashForRound"] = nil - header.Logs = append(header.Logs, newRoundEvent) - blockEvent := &blockwatch.Event{ - Type: blockwatch.Added, - BlockHeader: header, - } + // CurrentRoundStartBlock error + lpEth.Errors["CurrentRoundStartBlock"] = testErr + expErr = fmt.Sprintf("error fetching current round start block %v=", testErr) + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(tw) + assert.EqualError(err, expErr) + lpEth.Errors["CurrentRoundStartBlock"] = nil - go tw.Watch() - time.Sleep(2 * time.Millisecond) - // Check state initialization + // GetLastestBlock error + watcher.err = fmt.Errorf("GetLatestBlock error") + expErr = fmt.Sprintf("error fetching last seen block err=%v", watcher.err) + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(tw) + assert.EqualError(err, expErr) + watcher.err = nil + + // TranscoderPoolSize error + lpEth.Errors["GetTranscoderPoolSize"] = testErr + expErr = fmt.Sprintf("error fetching initial transcoderPoolSize err=%v", testErr) + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(tw) + assert.EqualError(err, expErr) + lpEth.Errors["GetTranscoderPoolSize"] = nil + + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(err) bh := tw.LastInitializedBlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(block, tw.LastSeenBlock()) - tw.Stop() // if watcher.GetLatestBlock() == nil, initialise lastSeenBlock to big.NewInt(0) watcher.latestHeader = nil tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) - go tw.Watch() - time.Sleep(2 * time.Millisecond) - // Check state initialization bh = tw.LastInitializedBlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(big.NewInt(0), tw.LastSeenBlock()) +} + +func TestTimeWatcher_WatchAndStop(t *testing.T) { + assert := assert.New(t) + size := big.NewInt(50) + block := big.NewInt(10) + round := big.NewInt(1) + hash := ethcommon.HexToHash("foo") + lpEth := ð.StubClient{ + PoolSize: size, + BlockNum: block, + BlockHashToReturn: hash, + Round: round, + Errors: make(map[string]error), + } + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: block}, + } + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + require.Nil(t, err) + + header := defaultMiniHeader() + newRoundEvent := newStubNewRoundLog() + + header.Logs = append(header.Logs, newRoundEvent) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + go tw.Watch() + time.Sleep(2 * time.Millisecond) // New Round event watcher.sink <- []*blockwatch.Event{blockEvent} @@ -123,7 +187,7 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) bhForRound = tw.LastInitializedBlockHash() - assert.Equal(hash, common.BytesToHash(bh[:])) + assert.Equal(hash, common.BytesToHash(bhForRound[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(header.Number, tw.LastSeenBlock()) @@ -132,16 +196,6 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { tw.Stop() time.Sleep(2 * time.Millisecond) assert.True(watcher.sub.unsubscribed) - - // Test watch error when RPC calls fail - tw = &TimeWatcher{ - lpEth: ð.StubClient{ - RoundsErr: errors.New("timewatcher error"), - }, - } - err = tw.Watch() - assert.NotNil(err) - assert.Contains(err.Error(), "timewatcher error") } func TestTimeWatcher_HandleLog(t *testing.T) { From 066a63137c9c04fdfcfd4e4efd0396717656eaf5 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 12 Apr 2021 17:12:54 +0200 Subject: [PATCH 23/72] eth: timewatcher code cleanup, remove fetchAndSetTranscoderPoolSize Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 3719aeed2f9dbf05cbba0c0d2873861f160cca73 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 19 +++++++------------ eth/watchers/timewatcher_test.go | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index c9fc6ca..11c09d2 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -71,7 +71,7 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l } num, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { - return nil, fmt.Errorf("error fetching current round start block %v=", err) + return nil, fmt.Errorf("error fetching current round start block err=%v", err) } tw.setLastInitializedRound(lr, bh, num) @@ -85,9 +85,11 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l } tw.setLastSeenBlock(blockNum) - if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { + size, err := tw.lpEth.GetTranscoderPoolSize() + if err != nil { return nil, fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) } + tw.setTranscoderPoolSize(size) return tw, nil } @@ -245,20 +247,13 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { } // Get the active transcoder pool size when we receive a NewRound event - if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { - return err - } - - tw.roundSubFeed.Send(log) - - return nil -} - -func (tw *TimeWatcher) fetchAndSetTranscoderPoolSize() error { size, err := tw.lpEth.GetTranscoderPoolSize() if err != nil { return err } tw.setTranscoderPoolSize(size) + + tw.roundSubFeed.Send(log) + return nil } diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index 72da00d..05a31c2 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -84,7 +84,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // CurrentRoundStartBlock error lpEth.Errors["CurrentRoundStartBlock"] = testErr - expErr = fmt.Sprintf("error fetching current round start block %v=", testErr) + expErr = fmt.Sprintf("error fetching current round start block err=%v", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) From e8f14b50fd1d591adf0b87693d8bb5a1a1b0e126 Mon Sep 17 00:00:00 2001 From: Ivan Tivonenko Date: Tue, 30 Nov 2021 19:45:56 +0200 Subject: [PATCH 24/72] quote errors Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: c2c73fa6f24473311766baee83e2762c87ef708d Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 2 +- eth/watchers/timewatcher.go | 10 +++++----- eth/watchers/timewatcher_test.go | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 87b7d56..6e23aad 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -121,7 +121,7 @@ func (w *Watcher) Watch(ctx context.Context) error { return nil case <-ticker.C: if err := w.syncToLatestBlock(); err != nil { - glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%v", err) + glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%q", err) } } } diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 11c09d2..7827e33 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -63,21 +63,21 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l lr, err := tw.lpEth.LastInitializedRound() if err != nil { - return nil, fmt.Errorf("error fetching initial lastInitializedRound value err=%v", err) + return nil, fmt.Errorf("error fetching initial lastInitializedRound value err=%q", err) } bh, err := tw.lpEth.BlockHashForRound(lr) if err != nil { - return nil, fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) + return nil, fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%q", err) } num, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { - return nil, fmt.Errorf("error fetching current round start block err=%v", err) + return nil, fmt.Errorf("error fetching current round start block err=%q", err) } tw.setLastInitializedRound(lr, bh, num) lastSeenBlock, err := tw.watcher.GetLatestBlock() if err != nil { - return nil, fmt.Errorf("error fetching last seen block err=%v", err) + return nil, fmt.Errorf("error fetching last seen block err=%q", err) } blockNum := big.NewInt(0) if lastSeenBlock != nil { @@ -87,7 +87,7 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l size, err := tw.lpEth.GetTranscoderPoolSize() if err != nil { - return nil, fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) + return nil, fmt.Errorf("error fetching initial transcoderPoolSize err=%q", err) } tw.setTranscoderPoolSize(size) diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index 05a31c2..a618719 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -68,7 +68,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // Last InitializedRound error lpEth.Errors["LastInitializedRound"] = testErr - expErr := fmt.Sprintf("error fetching initial lastInitializedRound value err=%v", testErr) + expErr := fmt.Sprintf("error fetching initial lastInitializedRound value err=%q", testErr) tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) @@ -76,7 +76,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // BlockHashForRound error lpEth.Errors["BlockHashForRound"] = testErr - expErr = fmt.Sprintf("error fetching initial lastInitializedBlockHash value err=%v", testErr) + expErr = fmt.Sprintf("error fetching initial lastInitializedBlockHash value err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) @@ -84,7 +84,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // CurrentRoundStartBlock error lpEth.Errors["CurrentRoundStartBlock"] = testErr - expErr = fmt.Sprintf("error fetching current round start block err=%v", testErr) + expErr = fmt.Sprintf("error fetching current round start block err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) @@ -92,7 +92,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // GetLastestBlock error watcher.err = fmt.Errorf("GetLatestBlock error") - expErr = fmt.Sprintf("error fetching last seen block err=%v", watcher.err) + expErr = fmt.Sprintf("error fetching last seen block err=%q", watcher.err) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) @@ -100,7 +100,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // TranscoderPoolSize error lpEth.Errors["GetTranscoderPoolSize"] = testErr - expErr = fmt.Sprintf("error fetching initial transcoderPoolSize err=%v", testErr) + expErr = fmt.Sprintf("error fetching initial transcoderPoolSize err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) From 9e45b661d1a86133d96ad5f6ea3b91f07d835581 Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Thu, 20 Jan 2022 15:55:03 +0100 Subject: [PATCH 25/72] eth: Handle watcher feeds asynchronously Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 87c6e404c7a8bd5552e66e346d3659d696e17f4f Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 7827e33..a6ed4f4 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -161,7 +161,9 @@ func (tw *TimeWatcher) Watch() error { case err := <-sub.Err(): glog.Error(err) case events := <-events: - tw.handleBlockEvents(events) + go func() { + tw.handleBlockEvents(events) + }() } } } From 7dbc72d16b2d98c0828e8f84b5ecebdf591ea96a Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Fri, 21 Jan 2022 09:11:42 +0100 Subject: [PATCH 26/72] eth: Refactor sink naming in the watcher feeds Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: f30819e43a6b0d9272301a402220d7df64d70063 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index a6ed4f4..5207755 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -151,8 +151,8 @@ func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { // Watch the blockwatch subscription for NewRound events func (tw *TimeWatcher) Watch() error { - events := make(chan []*blockwatch.Event, 10) - sub := tw.watcher.Subscribe(events) + blockSink := make(chan []*blockwatch.Event, 10) + sub := tw.watcher.Subscribe(blockSink) defer sub.Unsubscribe() for { select { @@ -160,9 +160,9 @@ func (tw *TimeWatcher) Watch() error { return nil case err := <-sub.Err(): glog.Error(err) - case events := <-events: + case block := <-blockSink: go func() { - tw.handleBlockEvents(events) + tw.handleBlockEvents(block) }() } } From b8763bdf521d54fa859ee48c671e88e1e6ce488b Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Tue, 25 Jan 2022 10:39:51 +0100 Subject: [PATCH 27/72] eth: Add locks to all feed receiver operations Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 39585360f2cd0c0626ad483e9dd677efb6d09387 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 5207755..496c508 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -39,6 +39,7 @@ type TimeWatcher struct { // last seen block number subscription feeds blockSubFeed event.Feed blockSubScope event.SubscriptionScope + feedMu sync.Mutex watcher BlockWatcher lpEth eth.LivepeerEthClient @@ -206,6 +207,9 @@ func (tw *TimeWatcher) handleBlockEvents(events []*blockwatch.Event) { } func (tw *TimeWatcher) handleBlockNum(event *blockwatch.Event) { + tw.feedMu.Lock() + defer tw.feedMu.Unlock() + last := tw.LastSeenBlock() new := event.BlockHeader.Number if last == nil || last.Cmp(new) != 0 { @@ -221,6 +225,9 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return nil } + tw.feedMu.Lock() + defer tw.feedMu.Unlock() + if eventName != "NewRound" { return fmt.Errorf("eventName is not NewRound") } From e3970e3e641da350fff437402256d18abac81970 Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Thu, 27 Jan 2022 15:48:47 +0100 Subject: [PATCH 28/72] eth: Refactor go func() {} into go Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 754c1ed203539275332363eb2cd2940a7fa6c72a Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 496c508..f79e686 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -162,9 +162,7 @@ func (tw *TimeWatcher) Watch() error { case err := <-sub.Err(): glog.Error(err) case block := <-blockSink: - go func() { - tw.handleBlockEvents(block) - }() + go tw.handleBlockEvents(block) } } } From e8f51fd7c0c894aa4970b281604cdcf67f630548 Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Mon, 31 Jan 2022 16:55:09 +0100 Subject: [PATCH 29/72] eth,pm,server: Use L1 block number for Ticket Parameters and Round Initialization Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 4b6ede31f040a084ff9e55bd798a0d6fffee1e1b Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 42 ++++++++++++++-- eth/blockwatch/block_watcher_test.go | 73 +++++++++++++++++++++++++++ eth/watchers/timewatcher.go | 74 ++++++++++++++-------------- eth/watchers/timewatcher_test.go | 60 +++++++++++----------- 4 files changed, 179 insertions(+), 70 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 6e23aad..b89fc8d 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -97,7 +97,7 @@ func (w *Watcher) BackfillEventsIfNeeded(ctx context.Context) error { return err } if len(events) > 0 { - w.blockFeed.Send(events) + w.blockFeed.Send(w.enrichWithL1(events)) } return nil } @@ -137,7 +137,11 @@ func (w *Watcher) Subscribe(sink chan<- []*Event) event.Subscription { // GetLatestBlock returns the latest block processed func (w *Watcher) GetLatestBlock() (*MiniHeader, error) { - return w.stack.Peek() + h, err := w.stack.Peek() + if err != nil { + return nil, err + } + return w.enrichWithL1BlockNumber(h) } // InspectRetainedBlocks returns the blocks retained in-memory by the Watcher instance. It is not @@ -202,7 +206,7 @@ func (w *Watcher) pollNextBlock() error { // Even if an error occurred, we still want to emit the events gathered since we might have // popped blocks off the Stack and they won't be re-added if len(events) != 0 { - w.blockFeed.Send(events) + w.blockFeed.Send(w.enrichWithL1(events)) } if err != nil { return err @@ -609,6 +613,38 @@ func (w *Watcher) filterLogsRecursively(from, to int, allLogs []types.Log) ([]ty return allLogs, nil } +// enrichWithL1 adds L1 block number to the event with the highest block number. +// +// Adding L1 block number to an event requires an RPC call and the only code using L1 block number is TimeWatcher that +// is interested only in the highest L1 block number, not in all of them. +func (w *Watcher) enrichWithL1(events []*Event) []*Event { + if len(events) == 0 { + return events + } + + max := 0 + for i, e := range events { + if events[max].BlockHeader.Number.Cmp(e.BlockHeader.Number) <= 0 { + max = i + } + } + + block, err := w.enrichWithL1BlockNumber(events[max].BlockHeader) + if err != nil { + glog.Errorf("Cannot fetch L1 block number for block number %d err=%q", events[max].BlockHeader.Number, err) + } + + events[max].BlockHeader = block + return events +} + +func (w *Watcher) enrichWithL1BlockNumber(header *MiniHeader) (*MiniHeader, error) { + if header == nil || header.L1BlockNumber != nil { + return header, nil + } + return w.client.HeaderByHash(header.Hash) +} + func isUnknownBlockErr(err error) bool { // Geth error if err.Error() == "unknown block" { diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 06e4311..3e4b88e 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -551,3 +551,76 @@ func aRange(from, to int) string { r := fmt.Sprintf("%d-%d", from, to) return r } + +func TestEnrichWithL1_Empty(t *testing.T) { + assert := assert.New(t) + + w := New(config) + res := w.enrichWithL1([]*Event{}) + assert.NotNil(res) + assert.Empty(res) +} + +func TestEnrichWithL1_One(t *testing.T) { + assert := assert.New(t) + + config.Client = &fakeClient{} + w := New(config) + header := &MiniHeader{ + Number: big.NewInt(FakeBlockNumber), + Hash: common.HexToHash(FakeHash), + } + events := []*Event{{BlockHeader: header}} + + res := w.enrichWithL1(events) + + assert.Len(res, 1) + assert.Equal(common.HexToHash(FakeHash), res[0].BlockHeader.Hash) + assert.Equal(big.NewInt(FakeBlockNumber), res[0].BlockHeader.Number) + assert.Equal(big.NewInt(FakeL1BlockNumber), res[0].BlockHeader.L1BlockNumber) +} + +func TestEnrichWithL1_Multiple(t *testing.T) { + assert := assert.New(t) + + config.Client = &fakeClient{} + w := New(config) + highestHeader := &MiniHeader{ + Number: big.NewInt(FakeBlockNumber), + Hash: common.HexToHash(FakeHash), + } + events := []*Event{ + {BlockHeader: &MiniHeader{Number: big.NewInt(1)}}, + {BlockHeader: highestHeader}, + {BlockHeader: &MiniHeader{Number: big.NewInt(2)}}, + } + + res := w.enrichWithL1(events) + + assert.Len(res, 3) + // highest block number event has L1 block number + assert.Equal(common.HexToHash(FakeHash), res[1].BlockHeader.Hash) + assert.Equal(big.NewInt(FakeBlockNumber), res[1].BlockHeader.Number) + assert.Equal(big.NewInt(FakeL1BlockNumber), res[1].BlockHeader.L1BlockNumber) + // other events do not have L1 block numbers + assert.Nil(res[0].BlockHeader.L1BlockNumber) + assert.Nil(res[2].BlockHeader.L1BlockNumber) +} + +func TestEnrichWithL1BlockNumber(t *testing.T) { + assert := assert.New(t) + + config.Client = &fakeClient{} + w := New(config) + header := &MiniHeader{ + Number: big.NewInt(FakeBlockNumber), + Hash: common.HexToHash(FakeHash), + } + + res, err := w.enrichWithL1BlockNumber(header) + + assert.NoError(err) + assert.Equal(common.HexToHash(FakeHash), res.Hash) + assert.Equal(big.NewInt(FakeBlockNumber), res.Number) + assert.Equal(big.NewInt(FakeL1BlockNumber), res.L1BlockNumber) +} diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index f79e686..28f8322 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -26,20 +26,20 @@ import ( // * Last Seen Block Number type TimeWatcher struct { // state - mu sync.RWMutex - lastInitializedRound *big.Int - lastInitializedBlockHash [32]byte - currentRoundStartBlock *big.Int - transcoderPoolSize *big.Int - lastSeenBlock *big.Int + mu sync.RWMutex + lastInitializedRound *big.Int + lastInitializedL1BlockHash [32]byte + currentRoundStartL1Block *big.Int + transcoderPoolSize *big.Int + lastSeenL1Block *big.Int // last initialized round number subscription feeds roundSubFeed event.Feed roundSubScope event.SubscriptionScope // last seen block number subscription feeds - blockSubFeed event.Feed - blockSubScope event.SubscriptionScope - feedMu sync.Mutex + l1BlockSubFeed event.Feed + l1BlockSubScope event.SubscriptionScope + feedMu sync.Mutex watcher BlockWatcher lpEth eth.LivepeerEthClient @@ -68,7 +68,7 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l } bh, err := tw.lpEth.BlockHashForRound(lr) if err != nil { - return nil, fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%q", err) + return nil, fmt.Errorf("error fetching initial lastInitializedL1BlockHash value err=%q", err) } num, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { @@ -80,11 +80,11 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l if err != nil { return nil, fmt.Errorf("error fetching last seen block err=%q", err) } - blockNum := big.NewInt(0) + l1BlockNum := big.NewInt(0) if lastSeenBlock != nil { - blockNum = lastSeenBlock.Number + l1BlockNum = lastSeenBlock.L1BlockNumber } - tw.setLastSeenBlock(blockNum) + tw.setLastSeenL1Block(l1BlockNum) size, err := tw.lpEth.GetTranscoderPoolSize() if err != nil { @@ -103,24 +103,24 @@ func (tw *TimeWatcher) LastInitializedRound() *big.Int { } // LastInitializedBlockHash returns the blockhash of the block the last round was initiated in -func (tw *TimeWatcher) LastInitializedBlockHash() [32]byte { +func (tw *TimeWatcher) LastInitializedL1BlockHash() [32]byte { tw.mu.RLock() defer tw.mu.RUnlock() - return tw.lastInitializedBlockHash + return tw.lastInitializedL1BlockHash } -func (tw *TimeWatcher) CurrentRoundStartBlock() *big.Int { +func (tw *TimeWatcher) CurrentRoundStartL1Block() *big.Int { tw.mu.RLock() defer tw.mu.RUnlock() - return tw.currentRoundStartBlock + return tw.currentRoundStartL1Block } func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, startBlk *big.Int) { tw.mu.Lock() defer tw.mu.Unlock() tw.lastInitializedRound = round - tw.lastInitializedBlockHash = hash - tw.currentRoundStartBlock = startBlk + tw.lastInitializedL1BlockHash = hash + tw.currentRoundStartL1Block = startBlk } func (tw *TimeWatcher) GetTranscoderPoolSize() *big.Int { @@ -138,16 +138,16 @@ func (tw *TimeWatcher) setTranscoderPoolSize(size *big.Int) { tw.transcoderPoolSize = size } -func (tw *TimeWatcher) LastSeenBlock() *big.Int { +func (tw *TimeWatcher) LastSeenL1Block() *big.Int { tw.mu.RLock() defer tw.mu.RUnlock() - return tw.lastSeenBlock + return tw.lastSeenL1Block } -func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { +func (tw *TimeWatcher) setLastSeenL1Block(blockNum *big.Int) { tw.mu.Lock() defer tw.mu.Unlock() - tw.lastSeenBlock = blockNum + tw.lastSeenL1Block = blockNum } // Watch the blockwatch subscription for NewRound events @@ -175,24 +175,24 @@ func (tw *TimeWatcher) SubscribeRounds(sink chan<- types.Log) event.Subscription return tw.roundSubScope.Track(tw.roundSubFeed.Subscribe(sink)) } -// SubscribeBlocks allows one to subscribe to newly seen block numbers +// SubscribeL1Blocks allows one to subscribe to newly seen L1 block numbers // To unsubscribe, simply call `Unsubscribe` on the returned subscription. // The sink channel should have ample buffer space to avoid blocking other subscribers. // Slow subscribers are not dropped. -func (tw *TimeWatcher) SubscribeBlocks(sink chan<- *big.Int) event.Subscription { - return tw.blockSubScope.Track(tw.blockSubFeed.Subscribe(sink)) +func (tw *TimeWatcher) SubscribeL1Blocks(sink chan<- *big.Int) event.Subscription { + return tw.l1BlockSubScope.Track(tw.l1BlockSubFeed.Subscribe(sink)) } // Stop TimeWatcher func (tw *TimeWatcher) Stop() { close(tw.quit) - tw.blockSubScope.Close() + tw.l1BlockSubScope.Close() tw.roundSubScope.Close() } func (tw *TimeWatcher) handleBlockEvents(events []*blockwatch.Event) { for _, event := range events { - tw.handleBlockNum(event) + tw.handleL1BlockNum(event) for _, log := range event.BlockHeader.Logs { if event.Type == blockwatch.Removed { log.Removed = true @@ -204,15 +204,15 @@ func (tw *TimeWatcher) handleBlockEvents(events []*blockwatch.Event) { } } -func (tw *TimeWatcher) handleBlockNum(event *blockwatch.Event) { +func (tw *TimeWatcher) handleL1BlockNum(event *blockwatch.Event) { tw.feedMu.Lock() defer tw.feedMu.Unlock() - last := tw.LastSeenBlock() - new := event.BlockHeader.Number - if last == nil || last.Cmp(new) != 0 { - tw.setLastSeenBlock(new) - tw.blockSubFeed.Send(new) + last := tw.LastSeenL1Block() + new := event.BlockHeader.L1BlockNumber + if new != nil && (last == nil || last.Cmp(new) != 0) { + tw.setLastSeenL1Block(new) + tw.l1BlockSubFeed.Send(new) } } @@ -235,7 +235,7 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return fmt.Errorf("unable to decode event: %v", err) } - roundStartBlock, err := tw.lpEth.CurrentRoundStartBlock() + roundStartL1Block, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { return err } @@ -248,9 +248,9 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { if err != nil { return err } - tw.setLastInitializedRound(lr, bh, roundStartBlock) + tw.setLastInitializedRound(lr, bh, roundStartL1Block) } else { - tw.setLastInitializedRound(nr.Round, nr.BlockHash, roundStartBlock) + tw.setLastInitializedRound(nr.Round, nr.BlockHash, roundStartL1Block) } // Get the active transcoder pool size when we receive a NewRound event diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index a618719..73f6123 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -24,13 +24,13 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { num := big.NewInt(10) tw.setLastInitializedRound(round, hash, num) assert.Equal(tw.lastInitializedRound, round) - assert.Equal(tw.lastInitializedBlockHash, hash) + assert.Equal(tw.lastInitializedL1BlockHash, hash) r := tw.LastInitializedRound() assert.Equal(r, round) - h := tw.LastInitializedBlockHash() + h := tw.LastInitializedL1BlockHash() assert.Equal(h, hash) - assert.Equal(tw.CurrentRoundStartBlock(), num) + assert.Equal(tw.CurrentRoundStartL1Block(), num) } func TestSetAndGet_TranscoderPoolSize(t *testing.T) { @@ -61,7 +61,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { Errors: make(map[string]error), } watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: block}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: block}, } testErr := fmt.Errorf("error") @@ -76,13 +76,13 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // BlockHashForRound error lpEth.Errors["BlockHashForRound"] = testErr - expErr = fmt.Sprintf("error fetching initial lastInitializedBlockHash value err=%q", testErr) + expErr = fmt.Sprintf("error fetching initial lastInitializedL1BlockHash value err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) lpEth.Errors["BlockHashForRound"] = nil - // CurrentRoundStartBlock error + // CurrentRoundStartL1Block error lpEth.Errors["CurrentRoundStartBlock"] = testErr expErr = fmt.Sprintf("error fetching current round start block err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) @@ -108,21 +108,21 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) - bh := tw.LastInitializedBlockHash() + bh := tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) - assert.Equal(block, tw.LastSeenBlock()) + assert.Equal(block, tw.LastSeenL1Block()) - // if watcher.GetLatestBlock() == nil, initialise lastSeenBlock to big.NewInt(0) + // if watcher.GetLatestBlock() == nil, initialise lastSeenL1Block to big.NewInt(0) watcher.latestHeader = nil tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) - bh = tw.LastInitializedBlockHash() + bh = tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) - assert.Equal(big.NewInt(0), tw.LastSeenBlock()) + assert.Equal(big.NewInt(0), tw.LastSeenL1Block()) } func TestTimeWatcher_WatchAndStop(t *testing.T) { @@ -139,7 +139,7 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { Errors: make(map[string]error), } watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: block}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: block}, } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) require.Nil(t, err) @@ -161,12 +161,12 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { time.Sleep(2 * time.Millisecond) lastRound := tw.LastInitializedRound() assert.Zero(lastRound.Cmp(big.NewInt(8))) - bhForRound := tw.LastInitializedBlockHash() + bhForRound := tw.LastInitializedL1BlockHash() var expectedHashForRound [32]byte copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) assert.Equal(size, tw.GetTranscoderPoolSize()) - assert.Equal(header.Number, tw.LastSeenBlock()) + assert.Equal(header.L1BlockNumber, tw.LastSeenL1Block()) // Test no NewRound events, values on rw remain the same tw.setTranscoderPoolSize(big.NewInt(10)) @@ -175,22 +175,22 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { time.Sleep(2 * time.Millisecond) lastRound = tw.LastInitializedRound() assert.Zero(lastRound.Cmp(big.NewInt(8))) - bhForRound = tw.LastInitializedBlockHash() + bhForRound = tw.LastInitializedL1BlockHash() copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) assert.Equal(big.NewInt(10), tw.GetTranscoderPoolSize()) - assert.Equal(header.Number, tw.LastSeenBlock()) + assert.Equal(header.L1BlockNumber, tw.LastSeenL1Block()) // Test RPC paths (event removed) blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) blockEvent.Type = blockwatch.Removed watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) - bhForRound = tw.LastInitializedBlockHash() + bhForRound = tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bhForRound[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) - assert.Equal(header.Number, tw.LastSeenBlock()) + assert.Equal(header.L1BlockNumber, tw.LastSeenL1Block()) // Test Stop tw.Stop() @@ -213,7 +213,7 @@ func TestTimeWatcher_HandleLog(t *testing.T) { err = tw.handleLog(log) assert.Nil(err) assert.Nil(tw.LastInitializedRound()) - assert.Equal([32]byte{}, tw.LastInitializedBlockHash()) + assert.Equal([32]byte{}, tw.LastInitializedL1BlockHash()) } func TestLastSeenBlock(t *testing.T) { @@ -221,21 +221,21 @@ func TestLastSeenBlock(t *testing.T) { tw := &TimeWatcher{} block := big.NewInt(5) - tw.setLastSeenBlock(block) - tw.LastSeenBlock() - assert.Equal(big.NewInt(5), tw.LastSeenBlock()) + tw.setLastSeenL1Block(block) + tw.LastSeenL1Block() + assert.Equal(big.NewInt(5), tw.LastSeenL1Block()) } func TestHandleBlockNum(t *testing.T) { assert := assert.New(t) watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) header := defaultMiniHeader() - header.Number = big.NewInt(10) + header.L1BlockNumber = big.NewInt(10) blockEvent := &blockwatch.Event{ Type: blockwatch.Added, BlockHeader: header, @@ -246,26 +246,26 @@ func TestHandleBlockNum(t *testing.T) { watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) - assert.Equal(tw.LastSeenBlock(), header.Number) + assert.Equal(tw.LastSeenL1Block(), header.L1BlockNumber) } func TestSubscribeBlocks(t *testing.T) { assert := assert.New(t) watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) header := defaultMiniHeader() - header.Number = big.NewInt(10) + header.L1BlockNumber = big.NewInt(10) blockEvent := &blockwatch.Event{ Type: blockwatch.Added, BlockHeader: header, } events := make(chan *big.Int, 10) - sub := tw.SubscribeBlocks(events) + sub := tw.SubscribeL1Blocks(events) defer sub.Unsubscribe() go tw.Watch() @@ -274,13 +274,13 @@ func TestSubscribeBlocks(t *testing.T) { watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) update := <-events - assert.Equal(update, header.Number) + assert.Equal(update, header.L1BlockNumber) } func TestSubscribeRounds(t *testing.T) { assert := assert.New(t) watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) From 8f77af1942b497d20a29dac9ad9046ed1d78586c Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Fri, 4 Feb 2022 13:32:14 +0100 Subject: [PATCH 30/72] eth,cmd: Backfill always from the back last seen in DB (instead of the last initialization round block) Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: a75e8e26d8e3880724c80f212a2568febd243d52 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 7 +---- eth/blockwatch/block_watcher_test.go | 42 ---------------------------- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index b89fc8d..084676e 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -45,7 +45,6 @@ type Config struct { Store MiniHeaderStore PollingInterval time.Duration StartBlockDepth rpc.BlockNumber - BackfillStartBlock *big.Int BlockRetentionLimit int WithLogs bool Topics []common.Hash @@ -58,7 +57,6 @@ type Config struct { type Watcher struct { blockRetentionLimit int startBlockDepth rpc.BlockNumber - backfillStartBlock *big.Int stack *Stack client Client blockFeed event.Feed @@ -79,7 +77,6 @@ func New(config Config) *Watcher { pollingInterval: config.PollingInterval, blockRetentionLimit: config.BlockRetentionLimit, startBlockDepth: config.StartBlockDepth, - backfillStartBlock: config.BackfillStartBlock, stack: stack, client: config.Client, withLogs: config.WithLogs, @@ -330,9 +327,7 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro } latestBlockNum := int(latestBlock.Number.Int64()) - if w.backfillStartBlock != nil { - startBlockNum = int(w.backfillStartBlock.Int64()) - } else if latestRetainedBlock != nil { + if latestRetainedBlock != nil { latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) // Events for latestRetainedBlock already processed, start at latestRetainedBlock + 1 startBlockNum = latestRetainedBlockNum + 1 diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 3e4b88e..7fc05df 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -214,48 +214,6 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { assert.Equal(t, big.NewInt(30), headers[0].Number) } -func TestGetMissedEventsToBackfill_BackfillStartBlock(t *testing.T) { - // Fixture will return block 30 as the tip of the chain - fakeClient, err := newFakeClient("testdata/fake_client_fast_sync_fixture.json") - require.NoError(t, err) - - store := &stubMiniHeaderStore{} - - config.Store = store - config.Client = fakeClient - config.BackfillStartBlock = big.NewInt(0) - watcher := New(config) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - events, err := watcher.getMissedEventsToBackfill(ctx) - require.NoError(t, err) - assert.Len(t, events, 1) - - // Check that block 30 is now in the DB as it is the last processed block - headers, err := store.FindAllMiniHeadersSortedByNumber() - require.NoError(t, err) - require.Len(t, headers, 1) - assert.Equal(t, big.NewInt(30), headers[0].Number) - - store = &stubMiniHeaderStore{} - config.Store = store - config.BackfillStartBlock = big.NewInt(5) - watcher = New(config) - events, err = watcher.getMissedEventsToBackfill(ctx) - require.NoError(t, err) - assert.Len(t, events, 1) - - // Check that block 30 is now in the DB as it is the last processed block - headers, err = store.FindAllMiniHeadersSortedByNumber() - require.NoError(t, err) - require.Len(t, headers, 1) - assert.Equal(t, big.NewInt(30), headers[0].Number) - - config.BackfillStartBlock = nil -} - func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { // Fixture will return block 5 as the tip of the chain fakeClient, err := newFakeClient("testdata/fake_client_basic_fixture.json") From bd5392f492289c743530a1a866a65801397df60d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Wed, 16 Feb 2022 16:39:12 +0100 Subject: [PATCH 31/72] Fix nil pointer in the block header logs Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 17a93a67b9cf3409b47e4e01797b2416cb6d234c Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 7 ++++++- eth/blockwatch/block_watcher_test.go | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 084676e..c097b58 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -637,7 +637,12 @@ func (w *Watcher) enrichWithL1BlockNumber(header *MiniHeader) (*MiniHeader, erro if header == nil || header.L1BlockNumber != nil { return header, nil } - return w.client.HeaderByHash(header.Hash) + fetchedBlock, err := w.client.HeaderByHash(header.Hash) + if err != nil { + return header, err + } + header.L1BlockNumber = fetchedBlock.L1BlockNumber + return header, nil } func isUnknownBlockErr(err error) bool { diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 7fc05df..f94f68b 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -527,6 +527,7 @@ func TestEnrichWithL1_One(t *testing.T) { header := &MiniHeader{ Number: big.NewInt(FakeBlockNumber), Hash: common.HexToHash(FakeHash), + Logs: []types.Log{logStub}, } events := []*Event{{BlockHeader: header}} @@ -536,6 +537,7 @@ func TestEnrichWithL1_One(t *testing.T) { assert.Equal(common.HexToHash(FakeHash), res[0].BlockHeader.Hash) assert.Equal(big.NewInt(FakeBlockNumber), res[0].BlockHeader.Number) assert.Equal(big.NewInt(FakeL1BlockNumber), res[0].BlockHeader.L1BlockNumber) + assert.Contains(res[0].BlockHeader.Logs, logStub) } func TestEnrichWithL1_Multiple(t *testing.T) { From e68c8ae5d9228683dc3b25c410edcaf9bdf0d6ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Wed, 13 Apr 2022 15:52:13 +0200 Subject: [PATCH 32/72] eth, pm: Backdate tickets by one round if the block hash for the current round is 0 (#2362) Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 11b0600fc76595799205a1229614a1dce954d787 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 56 ++++++++++++++++---- eth/watchers/timewatcher_test.go | 90 +++++++++++++++++++++++++++++--- 2 files changed, 129 insertions(+), 17 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 28f8322..4fbcd24 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -26,12 +26,13 @@ import ( // * Last Seen Block Number type TimeWatcher struct { // state - mu sync.RWMutex - lastInitializedRound *big.Int - lastInitializedL1BlockHash [32]byte - currentRoundStartL1Block *big.Int - transcoderPoolSize *big.Int - lastSeenL1Block *big.Int + mu sync.RWMutex + lastInitializedRound *big.Int + lastInitializedL1BlockHash [32]byte + preLastInitializedL1BlockHash [32]byte + currentRoundStartL1Block *big.Int + transcoderPoolSize *big.Int + lastSeenL1Block *big.Int // last initialized round number subscription feeds roundSubFeed event.Feed @@ -70,11 +71,15 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l if err != nil { return nil, fmt.Errorf("error fetching initial lastInitializedL1BlockHash value err=%q", err) } + pbh, err := tw.lpEth.BlockHashForRound(prevRound(lr)) + if err != nil { + return nil, fmt.Errorf("error fetching initial preLastInitializedL1BlockHash value err=%q", err) + } num, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { return nil, fmt.Errorf("error fetching current round start block err=%q", err) } - tw.setLastInitializedRound(lr, bh, num) + tw.setLastInitializedRound(lr, bh, pbh, num) lastSeenBlock, err := tw.watcher.GetLatestBlock() if err != nil { @@ -102,24 +107,32 @@ func (tw *TimeWatcher) LastInitializedRound() *big.Int { return tw.lastInitializedRound } -// LastInitializedBlockHash returns the blockhash of the block the last round was initiated in +// LastInitializedL1BlockHash returns the blockhash of the L1 block the last round was initiated in func (tw *TimeWatcher) LastInitializedL1BlockHash() [32]byte { tw.mu.RLock() defer tw.mu.RUnlock() return tw.lastInitializedL1BlockHash } +// PreLastInitializedL1BlockHash returns the blockhash of the L1 block for the round proceeding the last initialized round +func (tw *TimeWatcher) PreLastInitializedL1BlockHash() [32]byte { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.preLastInitializedL1BlockHash +} + func (tw *TimeWatcher) CurrentRoundStartL1Block() *big.Int { tw.mu.RLock() defer tw.mu.RUnlock() return tw.currentRoundStartL1Block } -func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, startBlk *big.Int) { +func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, preHash [32]byte, startBlk *big.Int) { tw.mu.Lock() defer tw.mu.Unlock() tw.lastInitializedRound = round tw.lastInitializedL1BlockHash = hash + tw.preLastInitializedL1BlockHash = preHash tw.currentRoundStartL1Block = startBlk } @@ -235,6 +248,10 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return fmt.Errorf("unable to decode event: %v", err) } + return tw.handleDecodedLog(log, nr) +} + +func (tw *TimeWatcher) handleDecodedLog(log types.Log, nr contracts.RoundsManagerNewRound) error { roundStartL1Block, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { return err @@ -248,9 +265,22 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { if err != nil { return err } - tw.setLastInitializedRound(lr, bh, roundStartL1Block) + pbh, err := tw.lpEth.BlockHashForRound(prevRound(lr)) + if err != nil { + return err + } + tw.setLastInitializedRound(lr, bh, pbh, roundStartL1Block) } else { - tw.setLastInitializedRound(nr.Round, nr.BlockHash, roundStartL1Block) + var pbh [32]byte + if nr.Round.Cmp(big.NewInt(0).Add(tw.lastInitializedRound, big.NewInt(1))) == 0 { + pbh = tw.lastInitializedL1BlockHash + } else { + pbh, err = tw.lpEth.BlockHashForRound(prevRound(nr.Round)) + if err != nil { + return err + } + } + tw.setLastInitializedRound(nr.Round, nr.BlockHash, pbh, roundStartL1Block) } // Get the active transcoder pool size when we receive a NewRound event @@ -264,3 +294,7 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return nil } + +func prevRound(lr *big.Int) *big.Int { + return big.NewInt(0).Sub(lr, big.NewInt(1)) +} diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index 73f6123..520b780 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -2,6 +2,7 @@ package watchers import ( "fmt" + "github.com/livepeer/go-livepeer/eth/contracts" "math/big" "testing" "time" @@ -21,15 +22,20 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { round := big.NewInt(5) var hash [32]byte copy(hash[:], "hello world") + var preHash [32]byte + copy(preHash[:], "hey hey hey") num := big.NewInt(10) - tw.setLastInitializedRound(round, hash, num) + tw.setLastInitializedRound(round, hash, preHash, num) assert.Equal(tw.lastInitializedRound, round) assert.Equal(tw.lastInitializedL1BlockHash, hash) + assert.Equal(tw.preLastInitializedL1BlockHash, preHash) r := tw.LastInitializedRound() assert.Equal(r, round) h := tw.LastInitializedL1BlockHash() assert.Equal(h, hash) + ph := tw.PreLastInitializedL1BlockHash() + assert.Equal(ph, preHash) assert.Equal(tw.CurrentRoundStartL1Block(), num) } @@ -106,10 +112,14 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { assert.EqualError(err, expErr) lpEth.Errors["GetTranscoderPoolSize"] = nil + preHash := ethcommon.HexToHash("bar") + lpEth.BlockHashForRoundMap = map[int64]common.Hash{0: preHash, 1: hash} tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) bh := tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) + pbh := tw.PreLastInitializedL1BlockHash() + assert.Equal(preHash, common.BytesToHash(pbh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(block, tw.LastSeenL1Block()) @@ -120,6 +130,8 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { assert.Nil(err) bh = tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) + pbh = tw.PreLastInitializedL1BlockHash() + assert.Equal(preHash, common.BytesToHash(pbh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(big.NewInt(0), tw.LastSeenL1Block()) @@ -199,7 +211,7 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { } func TestTimeWatcher_HandleLog(t *testing.T) { - lpEth := ð.StubClient{} + lpEth := ð.StubClient{Round: big.NewInt(2)} watcher := &stubBlockWatcher{} tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) require.Nil(t, err) @@ -212,10 +224,76 @@ func TestTimeWatcher_HandleLog(t *testing.T) { err = tw.handleLog(log) assert.Nil(err) - assert.Nil(tw.LastInitializedRound()) + assert.Equal(big.NewInt(2), tw.LastInitializedRound()) assert.Equal([32]byte{}, tw.LastInitializedL1BlockHash()) } +func TestTimeWatcher_HandleDecodedLog(t *testing.T) { + assert := assert.New(t) + + lpEth := ð.StubClient{ + Round: big.NewInt(2), + BlockHashForRoundMap: map[int64]common.Hash{ + 1: ethcommon.BytesToHash([]byte("one")), + 2: ethcommon.BytesToHash([]byte("two")), + 3: ethcommon.BytesToHash([]byte("three")), + 4: ethcommon.BytesToHash([]byte("four")), + 5: ethcommon.BytesToHash([]byte("five")), + 6: ethcommon.BytesToHash([]byte("six")), + }, + } + watcher := &stubBlockWatcher{} + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + require.Nil(t, err) + + var hash [32]byte + var preHash [32]byte + var nr contracts.RoundsManagerNewRound + + // Test remove log, data refreshed from eth client + log := newStubBaseLog() + log.Removed = true + lpEth.Round = big.NewInt(3) + + err = tw.handleDecodedLog(log, nr) + + assert.Nil(err) + assert.Equal(big.NewInt(3), tw.LastInitializedRound()) + copy(hash[:], lpEth.BlockHashForRoundMap[3].Bytes()) + assert.Equal(hash, tw.LastInitializedL1BlockHash()) + copy(preHash[:], lpEth.BlockHashForRoundMap[2].Bytes()) + assert.Equal(preHash, tw.PreLastInitializedL1BlockHash()) + + // Test add log with subsequent round + log.Removed = false + currentHash := ethcommon.BytesToHash([]byte("last")) + tw.lastInitializedL1BlockHash = currentHash + nr.Round = big.NewInt(4) + nr.BlockHash = lpEth.BlockHashForRoundMap[4] + + err = tw.handleDecodedLog(log, nr) + + assert.Nil(err) + assert.Equal(big.NewInt(4), tw.LastInitializedRound()) + copy(hash[:], lpEth.BlockHashForRoundMap[4].Bytes()) + assert.Equal(hash, tw.LastInitializedL1BlockHash()) + copy(preHash[:], currentHash.Bytes()) + assert.Equal(preHash, tw.PreLastInitializedL1BlockHash()) + + // Test add log with not subsequent round + nr.Round = big.NewInt(6) + nr.BlockHash = lpEth.BlockHashForRoundMap[6] + + err = tw.handleDecodedLog(log, nr) + + assert.Nil(err) + assert.Equal(big.NewInt(6), tw.LastInitializedRound()) + copy(hash[:], lpEth.BlockHashForRoundMap[6].Bytes()) + assert.Equal(hash, tw.LastInitializedL1BlockHash()) + copy(preHash[:], lpEth.BlockHashForRoundMap[5].Bytes()) + assert.Equal(preHash, tw.PreLastInitializedL1BlockHash()) +} + func TestLastSeenBlock(t *testing.T) { assert := assert.New(t) tw := &TimeWatcher{} @@ -232,7 +310,7 @@ func TestHandleBlockNum(t *testing.T) { latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } - tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{Round: big.NewInt(2)}) assert.Nil(err) header := defaultMiniHeader() header.L1BlockNumber = big.NewInt(10) @@ -255,7 +333,7 @@ func TestSubscribeBlocks(t *testing.T) { latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } - tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{Round: big.NewInt(2)}) assert.Nil(err) header := defaultMiniHeader() header.L1BlockNumber = big.NewInt(10) @@ -283,7 +361,7 @@ func TestSubscribeRounds(t *testing.T) { latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } - tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{Round: big.NewInt(2)}) assert.Nil(err) header := defaultMiniHeader() newRoundEvent := newStubNewRoundLog() From 5b2c07c1ae20a7f6d08df182ec6e065f8b421d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Mon, 11 Jul 2022 12:23:51 +0200 Subject: [PATCH 33/72] eth: Backfill blocks in batches (#2489) Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 159deaa162065f933ac472d11b05510f829d1692 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 40 ++++++++++++++++------------ eth/blockwatch/block_watcher_test.go | 7 ++--- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index c097b58..bc93277 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -85,10 +85,12 @@ func New(config Config) *Watcher { return bs } -// BackfillEventsIfNeeded finds missed events that might have occured while the -// node was offline and sends them to event subscribers. It blocks until -// it is done backfilling or the given context is canceled. -func (w *Watcher) BackfillEventsIfNeeded(ctx context.Context) error { +// BackfillEvents finds missed events and sends them to event subscribers. +// It blocks until it is done backfilling or the given context is canceled. +// Note that the latest block is never backfilled here from logs. It will be polled separately in syncToLatestBlock(). +// The reason for that is that we always need to propagate events from the latest block even if it does not contain +// events which are filtered out during the backfilling process. +func (w *Watcher) BackfillEvents(ctx context.Context) error { events, err := w.getMissedEventsToBackfill(ctx) if err != nil { return err @@ -117,7 +119,7 @@ func (w *Watcher) Watch(ctx context.Context) error { ticker.Stop() return nil case <-ticker.C: - if err := w.syncToLatestBlock(); err != nil { + if err := w.syncToLatestBlock(ctx); err != nil { glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%q", err) } } @@ -147,9 +149,14 @@ func (w *Watcher) InspectRetainedBlocks() ([]*MiniHeader, error) { return w.stack.Inspect() } -func (w *Watcher) syncToLatestBlock() error { +func (w *Watcher) syncToLatestBlock(ctx context.Context) error { w.Lock() defer w.Unlock() + + if err := w.BackfillEvents(ctx); err != nil { + return err + } + newestHeader, err := w.client.HeaderByNumber(nil) if err != nil { return err @@ -303,10 +310,11 @@ func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { return header, nil } -// getMissedEventsToBackfill finds missed events that might have occured while the node was -// offline. It does this by comparing the last block stored with the latest block discoverable via RPC. -// If the stored block is older then the latest block, it batch fetches the events for missing blocks, +// getMissedEventsToBackfill finds missed events that might have occurred since the last block polling. +// It does this by comparing the last block stored with the latest block discoverable via RPC. +// If the stored block is older than the latest block, it batch-fetches the events for missing blocks, // re-sets the stored blocks and returns the block events found. +// Note that the latest block is never backfilled, and will be polled separately during in syncToLatestBlock(). func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, error) { events := []*Event{} @@ -325,7 +333,9 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro if err != nil { return events, err } - latestBlockNum := int(latestBlock.Number.Int64()) + + // Latest block will be polled separately in syncToLatestBlock(), so it's not backfilled. + preLatestBlockNum := int(latestBlock.Number.Int64()) - 1 if latestRetainedBlock != nil { latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) @@ -335,14 +345,11 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro return events, nil } - if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed <= 0 { + if blocksElapsed = preLatestBlockNum - startBlockNum; blocksElapsed <= 0 { return events, nil } - glog.Infof("Backfilling block events (this can take a while)...\n") - glog.Infof("Start block: %v End block: %v Blocks elapsed: %v\n", startBlockNum, startBlockNum+blocksElapsed, blocksElapsed) - - logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, latestBlockNum) + logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, preLatestBlockNum) if furthestBlockProcessed > latestRetainedBlockNum { // If we have processed blocks further then the latestRetainedBlock in the DB, we // want to remove all blocks from the DB and insert the furthestBlockProcessed @@ -396,7 +403,6 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro BlockHeader: blockHeader, }) } - glog.Info("Done backfilling block events") return events, nil } return events, nil @@ -558,7 +564,7 @@ func (w *Watcher) getSubBlockRanges(from, to, rangeSize int) []*blockRange { const infuraTooManyResultsErrMsg = "query returned more than 10000 results" func (w *Watcher) filterLogsRecursively(from, to int, allLogs []types.Log) ([]types.Log, error) { - glog.Infof("fetching block logs from=%v to=%v", from, to) + glog.V(6).Infof("Polling blocks from=%v to=%v", from, to) numBlocks := to - from topics := [][]common.Hash{} if len(w.topics) > 0 { diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index f94f68b..47ac137 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -189,12 +189,13 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { store := &stubMiniHeaderStore{} // Add block number 5 as the last block seen by BlockWatcher - lastBlockSeen := &MiniHeader{ + preLastBlockSeen := &MiniHeader{ Number: big.NewInt(5), Hash: common.HexToHash("0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f"), Parent: common.HexToHash("0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba"), } - err = store.InsertMiniHeader(lastBlockSeen) + err = store.InsertMiniHeader(preLastBlockSeen) + require.NoError(t, err) config.Store = store @@ -211,7 +212,7 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { headers, err := store.FindAllMiniHeadersSortedByNumber() require.NoError(t, err) require.Len(t, headers, 1) - assert.Equal(t, big.NewInt(30), headers[0].Number) + assert.Equal(t, big.NewInt(29), headers[0].Number) } func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { From bdf5252a63e2e0dbe041115e45c5d97bac21a8de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Thu, 23 Mar 2023 07:35:18 +0100 Subject: [PATCH 34/72] Reduce number of ETH RPC calls during block polling (#2775) Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 8c7ec0155c5ba040f7066ab0a30045c37f15be2a Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 181 +++++---------------------- eth/blockwatch/block_watcher_test.go | 17 ++- 2 files changed, 41 insertions(+), 157 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index bc93277..3f49ab2 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -20,7 +20,7 @@ import ( // a hard limit of 10,000 logs returned by a single `eth_getLogs` query by Infura's Ethereum nodes so // we need to try and stay below it. Parity, Geth and Alchemy all have much higher limits (if any) on // the number of logs returned so Infura is by far the limiting factor. -var maxBlocksInGetLogsQuery = 60 +var maxBlocksInGetLogsQuery = 1000 // EventType describes the types of events emitted by blockwatch.Watcher. A block can be discovered // and added to our representation of the chain. During a block re-org, a block previously stored @@ -90,8 +90,8 @@ func New(config Config) *Watcher { // Note that the latest block is never backfilled here from logs. It will be polled separately in syncToLatestBlock(). // The reason for that is that we always need to propagate events from the latest block even if it does not contain // events which are filtered out during the backfilling process. -func (w *Watcher) BackfillEvents(ctx context.Context) error { - events, err := w.getMissedEventsToBackfill(ctx) +func (w *Watcher) BackfillEvents(ctx context.Context, chainHead *MiniHeader) error { + var events, err = w.getMissedEventsToBackfill(ctx, chainHead) if err != nil { return err } @@ -153,146 +153,12 @@ func (w *Watcher) syncToLatestBlock(ctx context.Context) error { w.Lock() defer w.Unlock() - if err := w.BackfillEvents(ctx); err != nil { - return err - } - newestHeader, err := w.client.HeaderByNumber(nil) if err != nil { return err } - lastSeenHeader, err := w.stack.Peek() - if err != nil { - return err - } - - if lastSeenHeader == nil { - return w.pollNextBlock() - } - - for i := lastSeenHeader.Number; i.Cmp(newestHeader.Number) < 0; i = i.Add(i, big.NewInt(1)) { - if err := w.pollNextBlock(); err != nil { - return err - } - } - return nil -} - -// pollNextBlock polls for the next block header to be added to the block stack. -// If there are no blocks on the stack, it fetches the first block at the specified -// `startBlockDepth` supplied at instantiation. -func (w *Watcher) pollNextBlock() error { - var nextBlockNumber *big.Int - latestHeader, err := w.stack.Peek() - if err != nil { - return err - } - if latestHeader == nil { - if w.startBlockDepth == rpc.LatestBlockNumber { - nextBlockNumber = nil // Fetch latest block - } else { - nextBlockNumber = big.NewInt(int64(w.startBlockDepth)) - } - } else { - nextBlockNumber = big.NewInt(0).Add(latestHeader.Number, big.NewInt(1)) - } - nextHeader, err := w.client.HeaderByNumber(nextBlockNumber) - if err != nil { - if err == ethereum.NotFound { - return nil // Noop and wait next polling interval - } - return err - } - - events := []*Event{} - events, err = w.buildCanonicalChain(nextHeader, events) - // Even if an error occurred, we still want to emit the events gathered since we might have - // popped blocks off the Stack and they won't be re-added - if len(events) != 0 { - w.blockFeed.Send(w.enrichWithL1(events)) - } - if err != nil { - return err - } - return nil -} - -func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ([]*Event, error) { - latestHeader, err := w.stack.Peek() - if err != nil { - return nil, err - } - // Is the stack empty or is it the next block? - if latestHeader == nil || nextHeader.Parent == latestHeader.Hash { - nextHeader, err := w.addLogs(nextHeader) - if err != nil { - // Due to block re-orgs & Ethereum node services load-balancing requests across multiple nodes - // a block header might be returned, but when fetching it's logs, an "unknown block" error is - // returned. This is expected to happen sometimes, and we simply return the events gathered so - // far and pick back up where we left off on the next polling interval. - if isUnknownBlockErr(err) { - glog.V(5).Infof("missing logs for blockNumber=%v - fetching on next polling interval", nextHeader.Number) - return events, nil - } - return events, err - } - err = w.stack.Push(nextHeader) - if err != nil { - return events, err - } - events = append(events, &Event{ - Type: Added, - BlockHeader: nextHeader, - }) - return events, nil - } - - // Pop latestHeader from the stack. We already have a reference to it. - if _, err := w.stack.Pop(); err != nil { - return events, err - } - events = append(events, &Event{ - Type: Removed, - BlockHeader: latestHeader, - }) - - nextParentHeader, err := w.client.HeaderByHash(nextHeader.Parent) - if err != nil { - if err == ethereum.NotFound { - glog.V(5).Infof("block header not found blockHash=%v - fetching on next polling interval", nextHeader.Parent.Hex()) - // Noop and wait next polling interval. We remove the popped blocks - // and refetch them on the next polling interval. - return events, nil - } - return events, err - } - events, err = w.buildCanonicalChain(nextParentHeader, events) - if err != nil { - return events, err - } - nextHeader, err = w.addLogs(nextHeader) - if err != nil { - // Due to block re-orgs & Ethereum node services load-balancing requests across multiple nodes - // a block header might be returned, but when fetching it's logs, an "unknown block" error is - // returned. This is expected to happen sometimes, and we simply return the events gathered so - // far and pick back up where we left off on the next polling interval. - if isUnknownBlockErr(err) { - glog.V(5).Infof("missing logs for blockNumber=%v - fetching on next polling interval", nextHeader.Number) - return events, nil - } - return events, err - } - err = w.stack.Push(nextHeader) - if err != nil { - return events, err - } - events = append(events, &Event{ - Type: Added, - BlockHeader: nextHeader, - }) - - return events, nil + return w.BackfillEvents(ctx, newestHeader) } func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { @@ -315,7 +181,7 @@ func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { // If the stored block is older than the latest block, it batch-fetches the events for missing blocks, // re-sets the stored blocks and returns the block events found. // Note that the latest block is never backfilled, and will be polled separately during in syncToLatestBlock(). -func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, error) { +func (w *Watcher) getMissedEventsToBackfill(ctx context.Context, chainHead *MiniHeader) ([]*Event, error) { events := []*Event{} var ( @@ -329,27 +195,38 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro return events, err } - latestBlock, err := w.client.HeaderByNumber(nil) - if err != nil { - return events, err + if chainHead == nil { + var err error + chainHead, err = w.client.HeaderByNumber(nil) + if err != nil { + return events, err + } } - // Latest block will be polled separately in syncToLatestBlock(), so it's not backfilled. - preLatestBlockNum := int(latestBlock.Number.Int64()) - 1 + latestBlockNum := int(chainHead.Number.Int64()) if latestRetainedBlock != nil { latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) // Events for latestRetainedBlock already processed, start at latestRetainedBlock + 1 startBlockNum = latestRetainedBlockNum + 1 } else { + err = w.stack.Push(chainHead) + if err != nil { + return events, err + } + + events = append(events, &Event{ + Type: Added, + BlockHeader: chainHead, + }) return events, nil } - if blocksElapsed = preLatestBlockNum - startBlockNum; blocksElapsed <= 0 { + if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed <= 0 { return events, nil } - logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, preLatestBlockNum) + logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, latestBlockNum) if furthestBlockProcessed > latestRetainedBlockNum { // If we have processed blocks further then the latestRetainedBlock in the DB, we // want to remove all blocks from the DB and insert the furthestBlockProcessed @@ -365,7 +242,12 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro } } // Add furthest block processed into the DB - latestHeader, err := w.client.HeaderByNumber(big.NewInt(int64(furthestBlockProcessed))) + var latestHeader *MiniHeader + if chainHead.Number.Int64() == int64(furthestBlockProcessed) { + latestHeader = chainHead + } else { + latestHeader, err = w.client.HeaderByNumber(big.NewInt(int64(furthestBlockProcessed))) + } if err != nil { return events, err } @@ -374,6 +256,11 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro return events, err } + events = append(events, &Event{ + Type: Added, + BlockHeader: latestHeader, + }) + // If no logs found, noop if len(logs) == 0 { return events, nil diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 47ac137..12c41d0 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -41,9 +41,6 @@ func TestWatcher(t *testing.T) { for i := 0; i < fakeClient.NumberOfTimesteps(); i++ { scenarioLabel := fakeClient.GetScenarioLabel() - err := watcher.pollNextBlock() - require.NoError(t, err) - retainedBlocks, err := watcher.InspectRetainedBlocks() require.NoError(t, err) expectedRetainedBlocks := fakeClient.ExpectedRetainedBlocks() @@ -204,15 +201,15 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - events, err := watcher.getMissedEventsToBackfill(ctx) + events, err := watcher.getMissedEventsToBackfill(ctx, nil) require.NoError(t, err) - assert.Len(t, events, 1) + assert.Len(t, events, 2) // Check that block 30 is now in the DB, and block 5 was removed. headers, err := store.FindAllMiniHeadersSortedByNumber() require.NoError(t, err) require.Len(t, headers, 1) - assert.Equal(t, big.NewInt(29), headers[0].Number) + assert.Equal(t, big.NewInt(30), headers[0].Number) } func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { @@ -236,7 +233,7 @@ func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - events, err := watcher.getMissedEventsToBackfill(ctx) + events, err := watcher.getMissedEventsToBackfill(ctx, nil) require.NoError(t, err) assert.Len(t, events, 0) @@ -262,13 +259,13 @@ func TestGetMissedEventsToBackfill_NOOP(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - events, err := watcher.getMissedEventsToBackfill(ctx) + events, err := watcher.getMissedEventsToBackfill(ctx, nil) require.NoError(t, err) - assert.Len(t, events, 0) + assert.Len(t, events, 1) headers, err := store.FindAllMiniHeadersSortedByNumber() require.NoError(t, err) - require.Len(t, headers, 0) + require.Len(t, headers, 1) } var logStub = types.Log{ From 13d823a6e41fdf012662e59cc807f8ad38967d97 Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Tue, 6 Jun 2023 13:42:28 -0300 Subject: [PATCH 35/72] go.mod: Bump go and livepeer-data version (#2804) * go.mod: Update livepeer-data lib * server: Fix MetadataQueue type Type changed on livepeer-data version * go.mod: Bump go to 1.20 The upgrade indirectly added a module that requires 1.19 * changelog: Change changelog * Go 1.20 -> 1.20.4 * Get rid of go-version-file * Bump lint version * Set go version in another place * go fmt * Add glog goroutine to ignored leak ones * Change order of go mod command * Bump linter version --------- Co-authored-by: Thom Shutt Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 1b9da7f25fa8c6261e8c465ea7e5c75166d220b6 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 4fbcd24..45acfe2 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -22,8 +22,8 @@ import ( // TimeWatcher allows for subscriptions to certain data feeds using a caller provided sink channel // consumers of the TimeWatcher can subscribe to following data feeds: -// * Last Initialized Round Number -// * Last Seen Block Number +// - Last Initialized Round Number +// - Last Seen Block Number type TimeWatcher struct { // state mu sync.RWMutex From cbb1508495fd31c90b8031f244104af8847f269c Mon Sep 17 00:00:00 2001 From: studystill <137779852+studystill@users.noreply.github.com> Date: Tue, 22 Jul 2025 15:39:36 +0800 Subject: [PATCH 36/72] refactor: use the built-in max/min to simplify the code (#3577) Signed-off-by: studystill Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 3a69e439b60acbcfb8edfda15896fb1c25039958 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 3f49ab2..32a9772 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -433,10 +433,7 @@ func (w *Watcher) getSubBlockRanges(from, to, rangeSize int) []*blockRange { for i := 0; i < numChunks; i = i + 1 { fromIndex := i * rangeSize - toIndex := fromIndex + rangeSize - if toIndex > len(blocks) { - toIndex = len(blocks) - } + toIndex := min(fromIndex+rangeSize, len(blocks)) bs := blocks[fromIndex:toIndex] blockRange := &blockRange{ FromBlock: bs[0], From 29a179c663a9560a514dad7726700e459fa1975b Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Thu, 8 Aug 2019 15:45:05 -0400 Subject: [PATCH 37/72] eth: Port blockwatcher and tests Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 0acd88bbd8c555bebbfc4be9775ba32ed6619cf2 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 581 +++++++++++++++++++++++++++ eth/blockwatch/block_watcher_test.go | 487 ++++++++++++++++++++++ 2 files changed, 1068 insertions(+) create mode 100644 eth/blockwatch/block_watcher.go create mode 100644 eth/blockwatch/block_watcher_test.go diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go new file mode 100644 index 0000000..d2a0a00 --- /dev/null +++ b/eth/blockwatch/block_watcher.go @@ -0,0 +1,581 @@ +package blockwatch + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/rpc" + "github.com/golang/glog" +) + +// maxBlocksInGetLogsQuery is the max number of blocks to fetch logs for in a single query. There is +// a hard limit of 10,000 logs returned by a single `eth_getLogs` query by Infura's Ethereum nodes so +// we need to try and stay below it. Parity, Geth and Alchemy all have much higher limits (if any) on +// the number of logs returned so Infura is by far the limiting factor. +var maxBlocksInGetLogsQuery = 60 + +// EventType describes the types of events emitted by blockwatch.Watcher. A block can be discovered +// and added to our representation of the chain. During a block re-org, a block previously stored +// can be removed from the list. +type EventType int + +const ( + // Added indicates that an event was added to our representation of the chain + Added EventType = iota + // Removed indicates that an event was removed from our representation of the chain + Removed +) + +// Event describes a block event emitted by a Watcher +type Event struct { + Type EventType + BlockHeader *MiniHeader +} + +// Config holds some configuration options for an instance of BlockWatcher. +type Config struct { + Store MiniHeaderStore + PollingInterval time.Duration + StartBlockDepth rpc.BlockNumber + BlockRetentionLimit int + WithLogs bool + Topics []common.Hash + Client Client +} + +// Watcher maintains a consistent representation of the latest `blockRetentionLimit` blocks, +// handling block re-orgs and network disruptions gracefully. It can be started from any arbitrary +// block height, and will emit both block added and removed events. +type Watcher struct { + blockRetentionLimit int + startBlockDepth rpc.BlockNumber + stack *Stack + client Client + blockFeed event.Feed + blockScope event.SubscriptionScope // Subscription scope tracking current live listeners + wasStartedOnce bool // Whether the block watcher has previously been started + pollingInterval time.Duration + ticker *time.Ticker + withLogs bool + topics []common.Hash + mu sync.RWMutex +} + +// New creates a new Watcher instance. +func New(config Config) *Watcher { + stack := NewStack(config.Store, config.BlockRetentionLimit) + + bs := &Watcher{ + pollingInterval: config.PollingInterval, + blockRetentionLimit: config.BlockRetentionLimit, + startBlockDepth: config.StartBlockDepth, + stack: stack, + client: config.Client, + withLogs: config.WithLogs, + topics: config.Topics, + } + return bs +} + +// BackfillEventsIfNeeded finds missed events that might have occured while the +// node was offline and sends them to event subscribers. It blocks until +// it is done backfilling or the given context is canceled. +func (w *Watcher) BackfillEventsIfNeeded(ctx context.Context) error { + events, err := w.getMissedEventsToBackfill(ctx) + if err != nil { + return err + } + if len(events) > 0 { + w.blockFeed.Send(events) + } + return nil +} + +// Watch starts the Watcher. It will continuously look for new blocks and blocks +// until the given context is canceled. Typically, you want to call Watch inside a goroutine. +func (w *Watcher) Watch(ctx context.Context) error { + w.mu.Lock() + if w.wasStartedOnce { + w.mu.Unlock() + return errors.New("Can only start Watcher once per instance") + } + w.wasStartedOnce = true + w.mu.Unlock() + + ticker := time.NewTicker(w.pollingInterval) + for { + select { + case <-ctx.Done(): + ticker.Stop() + return nil + case <-ticker.C: + if err := w.pollNextBlock(); err != nil { + glog.Error("blockwatch.Watcher error encountered") + } + } + } +} + +// Subscribe allows one to subscribe to the block events emitted by the Watcher. +// To unsubscribe, simply call `Unsubscribe` on the returned subscription. +// The sink channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. +func (w *Watcher) Subscribe(sink chan<- []*Event) event.Subscription { + return w.blockScope.Track(w.blockFeed.Subscribe(sink)) +} + +// GetLatestBlock returns the latest block processed +func (w *Watcher) GetLatestBlock() (*MiniHeader, error) { + return w.stack.Peek() +} + +// InspectRetainedBlocks returns the blocks retained in-memory by the Watcher instance. It is not +// particularly performant and therefore should only be used for debugging and testing purposes. +func (w *Watcher) InspectRetainedBlocks() ([]*MiniHeader, error) { + return w.stack.Inspect() +} + +// pollNextBlock polls for the next block header to be added to the block stack. +// If there are no blocks on the stack, it fetches the first block at the specified +// `startBlockDepth` supplied at instantiation. +func (w *Watcher) pollNextBlock() error { + var nextBlockNumber *big.Int + latestHeader, err := w.stack.Peek() + if err != nil { + return err + } + if latestHeader == nil { + if w.startBlockDepth == rpc.LatestBlockNumber { + nextBlockNumber = nil // Fetch latest block + } else { + nextBlockNumber = big.NewInt(int64(w.startBlockDepth)) + } + } else { + nextBlockNumber = big.NewInt(0).Add(latestHeader.Number, big.NewInt(1)) + } + nextHeader, err := w.client.HeaderByNumber(nextBlockNumber) + if err != nil { + if err == ethereum.NotFound { + glog.Infof("block header not found blockNumber=%v", nextBlockNumber) + return nil // Noop and wait next polling interval + } + return err + } + + events := []*Event{} + events, err = w.buildCanonicalChain(nextHeader, events) + // Even if an error occurred, we still want to emit the events gathered since we might have + // popped blocks off the Stack and they won't be re-added + if len(events) != 0 { + w.blockFeed.Send(events) + } + if err != nil { + return err + } + return nil +} + +func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ([]*Event, error) { + latestHeader, err := w.stack.Peek() + if err != nil { + return nil, err + } + // Is the stack empty or is it the next block? + if latestHeader == nil || nextHeader.Parent == latestHeader.Hash { + nextHeader, err := w.addLogs(nextHeader) + if err != nil { + // Due to block re-orgs & Ethereum node services load-balancing requests across multiple nodes + // a block header might be returned, but when fetching it's logs, an "unknown block" error is + // returned. This is expected to happen sometimes, and we simply return the events gathered so + // far and pick back up where we left off on the next polling interval. + if isUnknownBlockErr(err) { + glog.Infof("failed to get logs for block nextHeader=%v", nextHeader) + return events, nil + } + return events, err + } + err = w.stack.Push(nextHeader) + if err != nil { + return events, err + } + events = append(events, &Event{ + Type: Added, + BlockHeader: nextHeader, + }) + return events, nil + } + + // Pop latestHeader from the stack. We already have a reference to it. + if _, err := w.stack.Pop(); err != nil { + return events, err + } + events = append(events, &Event{ + Type: Removed, + BlockHeader: latestHeader, + }) + + nextParentHeader, err := w.client.HeaderByHash(nextHeader.Parent) + if err != nil { + if err == ethereum.NotFound { + glog.Infof("block header not found blockHash=%v", nextHeader.Parent.Hex()) + // Noop and wait next polling interval. We remove the popped blocks + // and refetch them on the next polling interval. + return events, nil + } + return events, err + } + events, err = w.buildCanonicalChain(nextParentHeader, events) + if err != nil { + return events, err + } + nextHeader, err = w.addLogs(nextHeader) + if err != nil { + // Due to block re-orgs & Ethereum node services load-balancing requests across multiple nodes + // a block header might be returned, but when fetching it's logs, an "unknown block" error is + // returned. This is expected to happen sometimes, and we simply return the events gathered so + // far and pick back up where we left off on the next polling interval. + if isUnknownBlockErr(err) { + glog.Infof("failed to get logs for block nextHeader=%v", nextHeader) + return events, nil + } + return events, err + } + err = w.stack.Push(nextHeader) + if err != nil { + return events, err + } + events = append(events, &Event{ + Type: Added, + BlockHeader: nextHeader, + }) + + return events, nil +} + +func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { + if !w.withLogs { + return header, nil + } + logs, err := w.client.FilterLogs(ethereum.FilterQuery{ + BlockHash: &header.Hash, + Topics: [][]common.Hash{w.topics}, + }) + if err != nil { + return header, err + } + header.Logs = logs + return header, nil +} + +// getMissedEventsToBackfill finds missed events that might have occured while the node was +// offline. It does this by comparing the last block stored with the latest block discoverable via RPC. +// If the stored block is older then the latest block, it batch fetches the events for missing blocks, +// re-sets the stored blocks and returns the block events found. +func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, error) { + events := []*Event{} + + latestRetainedBlock, err := w.stack.Peek() + if err != nil { + return events, err + } + // No blocks stored, nowhere to backfill to + if latestRetainedBlock == nil { + return events, nil + } + latestBlock, err := w.client.HeaderByNumber(nil) + if err != nil { + return events, err + } + blocksElapsed := big.NewInt(0).Sub(latestBlock.Number, latestRetainedBlock.Number) + if blocksElapsed.Int64() == 0 { + return events, nil + } + + glog.Infof("Some blocks have elapsed since last boot. Backfilling block events (this can take a while)... blocksElapsed=%v", blocksElapsed.Int64()) + startBlockNum := int(latestRetainedBlock.Number.Int64() + 1) + endBlockNum := int(latestRetainedBlock.Number.Int64() + blocksElapsed.Int64()) + logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, endBlockNum) + if int64(furthestBlockProcessed) > latestRetainedBlock.Number.Int64() { + // If we have processed blocks further then the latestRetainedBlock in the DB, we + // want to remove all blocks from the DB and insert the furthestBlockProcessed + // Doing so will cause the BlockWatcher to start from that furthestBlockProcessed. + headers, err := w.InspectRetainedBlocks() + if err != nil { + return events, err + } + for i := 0; i < len(headers); i++ { + _, err := w.stack.Pop() + if err != nil { + return events, err + } + } + // Add furthest block processed into the DB + latestHeader, err := w.client.HeaderByNumber(big.NewInt(int64(furthestBlockProcessed))) + if err != nil { + return events, err + } + err = w.stack.Push(latestHeader) + if err != nil { + return events, err + } + + // If no logs found, noop + if len(logs) == 0 { + return events, nil + } + + // Create the block events from all the logs found by grouping + // them into blockHeaders + hashToBlockHeader := map[common.Hash]*MiniHeader{} + for _, log := range logs { + blockHeader, ok := hashToBlockHeader[log.BlockHash] + if !ok { + // TODO(fabio): Find a way to include the parent hash for the block as well. + // It's currently not an issue to omit it since we don't use the parent hash + // when processing block events in OrderWatcher. + blockHeader = &MiniHeader{ + Hash: log.BlockHash, + Number: big.NewInt(0).SetUint64(log.BlockNumber), + Logs: []types.Log{}, + } + hashToBlockHeader[log.BlockHash] = blockHeader + } + blockHeader.Logs = append(blockHeader.Logs, log) + } + for _, blockHeader := range hashToBlockHeader { + events = append(events, &Event{ + Type: Added, + BlockHeader: blockHeader, + }) + } + glog.Info("Done backfilling block events") + return events, nil + } + return events, nil +} + +type logRequestResult struct { + From int + To int + Logs []types.Log + Err error +} + +// getLogsRequestChunkSize is the number of `eth_getLogs` JSON RPC to send concurrently in each batch fetch +const getLogsRequestChunkSize = 3 + +// getLogsInBlockRange attempts to fetch all logs in the block range supplied. It implements a +// limited-concurrency batch fetch, where all requests in the previous batch must complete for +// the next batch of requests to be sent. If an error is encountered in a batch, all subsequent +// batch requests are not sent. Instead, it returns all the logs it found up until the error was +// encountered, along with the block number after which no further logs were retrieved. +func (w *Watcher) getLogsInBlockRange(ctx context.Context, from, to int) ([]types.Log, int) { + blockRanges := w.getSubBlockRanges(from, to, maxBlocksInGetLogsQuery) + + numChunks := 0 + chunkChan := make(chan []*blockRange, 1000000) + for len(blockRanges) != 0 { + var chunk []*blockRange + if len(blockRanges) < getLogsRequestChunkSize { + chunk = blockRanges[:len(blockRanges)] + } else { + chunk = blockRanges[:getLogsRequestChunkSize] + } + chunkChan <- chunk + blockRanges = blockRanges[len(chunk):] + numChunks++ + } + + semaphoreChan := make(chan struct{}, 1) + defer close(semaphoreChan) + + didAPreviousRequestFail := false + furthestBlockProcessed := from - 1 + allLogs := []types.Log{} + + for i := 0; i < numChunks; i++ { + // Add one to the semaphore chan. If it already has a value, the chunk blocks here until one frees up. + // We deliberately process the chunks sequentially, since if any request results in an error, we + // do not want to send any further requests. + semaphoreChan <- struct{}{} + + // If a previous request failed, we stop processing newer requests + if didAPreviousRequestFail { + <-semaphoreChan + continue // Noop + } + + mu := sync.Mutex{} + indexToLogResult := map[int]logRequestResult{} + chunk := <-chunkChan + + wg := &sync.WaitGroup{} + for i, aBlockRange := range chunk { + wg.Add(1) + go func(index int, b *blockRange) { + defer wg.Done() + + select { + case <-ctx.Done(): + indexToLogResult[index] = logRequestResult{ + From: b.FromBlock, + To: b.ToBlock, + Err: errors.New("context was canceled"), + Logs: []types.Log{}, + } + return + default: + } + + logs, err := w.filterLogsRecursively(b.FromBlock, b.ToBlock, []types.Log{}) + if err != nil { + glog.Errorf("failed to fetch logs for range error=%v fromBlock=%v toBlock=%v", err, b.FromBlock, b.ToBlock) + } + mu.Lock() + indexToLogResult[index] = logRequestResult{ + From: b.FromBlock, + To: b.ToBlock, + Err: err, + Logs: logs, + } + mu.Unlock() + }(i, aBlockRange) + } + + // Wait for all log requests to complete + wg.Wait() + + for i, aBlockRange := range chunk { + logRequestResult := indexToLogResult[i] + // Break at first error encountered + if logRequestResult.Err != nil { + didAPreviousRequestFail = true + furthestBlockProcessed = logRequestResult.From - 1 + break + } + allLogs = append(allLogs, logRequestResult.Logs...) + furthestBlockProcessed = aBlockRange.ToBlock + } + <-semaphoreChan + } + + return allLogs, furthestBlockProcessed +} + +type blockRange struct { + FromBlock int + ToBlock int +} + +// getSubBlockRanges breaks up the block range into smaller block ranges of rangeSize. +// `eth_getLogs` requests are inclusive to both the start and end blocks specified and +// so we need to make the ranges exclusive of one another to avoid fetching the same +// blocks' logs twice. +func (w *Watcher) getSubBlockRanges(from, to, rangeSize int) []*blockRange { + chunks := []*blockRange{} + numBlocksLeft := to - from + if numBlocksLeft < rangeSize { + chunks = append(chunks, &blockRange{ + FromBlock: from, + ToBlock: to, + }) + } else { + blocks := []int{} + for i := 0; i <= numBlocksLeft; i++ { + blocks = append(blocks, from+i) + } + numChunks := len(blocks) / rangeSize + remainder := len(blocks) % rangeSize + if remainder > 0 { + numChunks = numChunks + 1 + } + + for i := 0; i < numChunks; i = i + 1 { + fromIndex := i * rangeSize + toIndex := fromIndex + rangeSize + if toIndex > len(blocks) { + toIndex = len(blocks) + } + bs := blocks[fromIndex:toIndex] + blockRange := &blockRange{ + FromBlock: bs[0], + ToBlock: bs[len(bs)-1], + } + chunks = append(chunks, blockRange) + } + } + return chunks +} + +const infuraTooManyResultsErrMsg = "query returned more than 10000 results" + +func (w *Watcher) filterLogsRecursively(from, to int, allLogs []types.Log) ([]types.Log, error) { + glog.Infof("fetching block logs from=%v to=%v", from, to) + numBlocks := to - from + topics := [][]common.Hash{} + if len(w.topics) > 0 { + topics = append(topics, w.topics) + } + logs, err := w.client.FilterLogs(ethereum.FilterQuery{ + FromBlock: big.NewInt(int64(from)), + ToBlock: big.NewInt(int64(to)), + Topics: topics, + }) + if err != nil { + // Infura caps the logs returned to 10,000 per request, if our request exceeds this limit, split it + // into two requests. Parity, Geth and Alchemy all have much higher limits (if any at all), so no need + // to expect any similar errors of this nature from them. + if err.Error() == infuraTooManyResultsErrMsg { + // HACK(fabio): Infura limits the returned results to 10,000 logs, BUT some single + // blocks contain more then 10,000 logs. This has supposedly been fixed but we keep + // this logic here just in case. It helps us avoid infinite recursion. + // Source: https://community.infura.io/t/getlogs-error-query-returned-more-than-1000-results/358/10 + if from == to { + return allLogs, fmt.Errorf("Unable to get the logs for block #%d, because it contains too many logs", from) + } + + r := numBlocks % 2 + firstBatchSize := numBlocks / 2 + secondBatchSize := firstBatchSize + if r == 1 { + secondBatchSize = secondBatchSize + 1 + } + + endFirstHalf := from + firstBatchSize + startSecondHalf := endFirstHalf + 1 + allLogs, err := w.filterLogsRecursively(from, endFirstHalf, allLogs) + if err != nil { + return nil, err + } + allLogs, err = w.filterLogsRecursively(startSecondHalf, to, allLogs) + if err != nil { + return nil, err + } + return allLogs, nil + } else { + return nil, err + } + } + allLogs = append(allLogs, logs...) + return allLogs, nil +} + +func isUnknownBlockErr(err error) bool { + // Geth error + if err.Error() == "unknown block" { + return true + } + // Parity error + if err.Error() == "One of the blocks specified in filter (fromBlock, toBlock or blockHash) cannot be found" { + return true + } + return false +} diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go new file mode 100644 index 0000000..4f11787 --- /dev/null +++ b/eth/blockwatch/block_watcher_test.go @@ -0,0 +1,487 @@ +package blockwatch + +import ( + "context" + "errors" + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var config = Config{ + PollingInterval: 1 * time.Second, + BlockRetentionLimit: 10, + StartBlockDepth: rpc.LatestBlockNumber, + WithLogs: false, + Topics: []common.Hash{}, +} + +var basicFakeClientFixture = "testdata/fake_client_basic_fixture.json" + +func TestWatcher(t *testing.T) { + fakeClient, err := newFakeClient("testdata/fake_client_block_poller_fixtures.json") + require.NoError(t, err) + + // Polling interval unused because we hijack the ticker for this test + config.Store = &stubMiniHeaderStore{} + config.Client = fakeClient + watcher := New(config) + + // Having a buffer of 1 unblocks the below for-loop without resorting to a goroutine + events := make(chan []*Event, 1) + sub := watcher.Subscribe(events) + + for i := 0; i < fakeClient.NumberOfTimesteps(); i++ { + scenarioLabel := fakeClient.GetScenarioLabel() + + err := watcher.pollNextBlock() + require.NoError(t, err) + + retainedBlocks, err := watcher.InspectRetainedBlocks() + require.NoError(t, err) + expectedRetainedBlocks := fakeClient.ExpectedRetainedBlocks() + assert.Equal(t, expectedRetainedBlocks, retainedBlocks, scenarioLabel) + + expectedEvents := fakeClient.GetEvents() + if len(expectedEvents) != 0 { + select { + case gotEvents := <-events: + assert.Equal(t, expectedEvents, gotEvents, scenarioLabel) + + case <-time.After(3 * time.Second): + t.Fatal("Timed out waiting for Events channel to deliver expected events") + } + } + + fakeClient.IncrementTimestep() + + if i == fakeClient.NumberOfTimesteps()-1 { + sub.Unsubscribe() + } + } +} + +func TestWatcherStartStop(t *testing.T) { + fakeClient, err := newFakeClient(basicFakeClientFixture) + require.NoError(t, err) + + config.Store = &stubMiniHeaderStore{} + config.Client = fakeClient + watcher := New(config) + + // Start the watcher in a goroutine. We use the done channel to signal when + // watcher.Watch returns. + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + defer cancel() + go func() { + require.NoError(t, watcher.Watch(ctx)) + done <- struct{}{} + }() + + // Wait a bit and then stop the watcher by calling cancel. + time.Sleep(100 * time.Millisecond) + cancel() + + // Make sure that the watcher actually stops. + select { + case <-time.After(100 * time.Millisecond): + t.Error("timed out waiting for watcher to stop") + case <-done: + break + } +} + +type blockRangeChunksTestCase struct { + from int + to int + expectedBlockRanges []*blockRange +} + +func TestGetSubBlockRanges(t *testing.T) { + rangeSize := 6 + testCases := []blockRangeChunksTestCase{ + blockRangeChunksTestCase{ + from: 10, + to: 10, + expectedBlockRanges: []*blockRange{ + &blockRange{ + FromBlock: 10, + ToBlock: 10, + }, + }, + }, + blockRangeChunksTestCase{ + from: 10, + to: 16, + expectedBlockRanges: []*blockRange{ + &blockRange{ + FromBlock: 10, + ToBlock: 15, + }, + &blockRange{ + FromBlock: 16, + ToBlock: 16, + }, + }, + }, + blockRangeChunksTestCase{ + from: 10, + to: 22, + expectedBlockRanges: []*blockRange{ + &blockRange{ + FromBlock: 10, + ToBlock: 15, + }, + &blockRange{ + FromBlock: 16, + ToBlock: 21, + }, + &blockRange{ + FromBlock: 22, + ToBlock: 22, + }, + }, + }, + blockRangeChunksTestCase{ + from: 10, + to: 24, + expectedBlockRanges: []*blockRange{ + &blockRange{ + FromBlock: 10, + ToBlock: 15, + }, + &blockRange{ + FromBlock: 16, + ToBlock: 21, + }, + &blockRange{ + FromBlock: 22, + ToBlock: 24, + }, + }, + }, + } + + fakeClient, err := newFakeClient(basicFakeClientFixture) + require.NoError(t, err) + config.Store = &stubMiniHeaderStore{} + config.Client = fakeClient + watcher := New(config) + + for _, testCase := range testCases { + blockRanges := watcher.getSubBlockRanges(testCase.from, testCase.to, rangeSize) + assert.Equal(t, testCase.expectedBlockRanges, blockRanges) + } +} + +func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { + // Fixture will return block 30 as the tip of the chain + fakeClient, err := newFakeClient("testdata/fake_client_fast_sync_fixture.json") + require.NoError(t, err) + + store := &stubMiniHeaderStore{} + // Add block number 5 as the last block seen by BlockWatcher + lastBlockSeen := &MiniHeader{ + Number: big.NewInt(5), + Hash: common.HexToHash("0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f"), + Parent: common.HexToHash("0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba"), + } + err = store.InsertMiniHeader(lastBlockSeen) + require.NoError(t, err) + + config.Store = store + config.Client = fakeClient + watcher := New(config) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events, err := watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 1) + + // Check that block 30 is now in the DB, and block 5 was removed. + headers, err := store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 1) + assert.Equal(t, big.NewInt(30), headers[0].Number) +} + +func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { + // Fixture will return block 5 as the tip of the chain + fakeClient, err := newFakeClient("testdata/fake_client_basic_fixture.json") + require.NoError(t, err) + + store := &stubMiniHeaderStore{} + // Add block number 5 as the last block seen by BlockWatcher + lastBlockSeen := &MiniHeader{ + Number: big.NewInt(5), + Hash: common.HexToHash("0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f"), + Parent: common.HexToHash("0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba"), + } + err = store.InsertMiniHeader(lastBlockSeen) + require.NoError(t, err) + + config.Store = store + config.Client = fakeClient + watcher := New(config) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events, err := watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 0) + + // Check that block 5 is still in the DB + headers, err := store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 1) + assert.Equal(t, big.NewInt(5), headers[0].Number) +} + +var logStub = types.Log{ + Address: common.HexToAddress("0x21ab6c9fac80c59d401b37cb43f81ea9dde7fe34"), + Topics: []common.Hash{ + common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), + common.HexToHash("0x0000000000000000000000004d8a4aa1f304f9632cf3877473445d85c577fe5d"), + common.HexToHash("0x0000000000000000000000004bdd0d16cfa18e33860470fc4d65c6f5cee60959"), + }, + Data: common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000337ad34c0"), + BlockNumber: uint64(30), + TxHash: common.HexToHash("0xd9bb5f9e888ee6f74bedcda811c2461230f247c205849d6f83cb6c3925e54586"), + TxIndex: uint(0), + BlockHash: common.HexToHash("0x6bbf9b6e836207ab25379c20e517a89090cbbaf8877746f6ed7fb6820770816b"), + Index: uint(0), + Removed: false, +} + +var errUnexpected = errors.New("Something unexpected") + +type filterLogsRecursivelyTestCase struct { + Label string + rangeToFilterLogsResponse map[string]filterLogsResponse + Err error + Logs []types.Log +} + +func TestFilterLogsRecursively(t *testing.T) { + from := 10 + to := 20 + testCases := []filterLogsRecursivelyTestCase{ + filterLogsRecursivelyTestCase{ + Label: "HAPPY_PATH", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub}, + }, + filterLogsRecursivelyTestCase{ + Label: "TOO_MANY_RESULTS_INFURA_ERROR", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Err: errors.New(infuraTooManyResultsErrMsg), + }, + "10-15": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + "16-20": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub, logStub}, + }, + filterLogsRecursivelyTestCase{ + Label: "TOO_MANY_RESULTS_INFURA_ERROR_DEEPER_RECURSION", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Err: errors.New(infuraTooManyResultsErrMsg), + }, + "10-15": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + "16-20": filterLogsResponse{ + Err: errors.New(infuraTooManyResultsErrMsg), + }, + "16-18": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + "19-20": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub, logStub, logStub}, + }, + filterLogsRecursivelyTestCase{ + Label: "TOO_MANY_RESULTS_INFURA_ERROR_DEEPER_RECURSION_FAILURE", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Err: errors.New(infuraTooManyResultsErrMsg), + }, + "10-15": filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + "16-20": filterLogsResponse{ + Err: errUnexpected, + }, + }, + Err: errUnexpected, + }, + filterLogsRecursivelyTestCase{ + Label: "UNEXPECTED_ERROR", + rangeToFilterLogsResponse: map[string]filterLogsResponse{ + "10-20": filterLogsResponse{ + Err: errUnexpected, + }, + }, + Err: errUnexpected, + }, + } + + config.Store = &stubMiniHeaderStore{} + + for _, testCase := range testCases { + fakeLogClient, err := newFakeLogClient(testCase.rangeToFilterLogsResponse) + require.NoError(t, err) + config.Client = fakeLogClient + watcher := New(config) + + logs, err := watcher.filterLogsRecursively(from, to, []types.Log{}) + require.Equal(t, testCase.Err, err, testCase.Label) + require.Equal(t, testCase.Logs, logs, testCase.Label) + assert.Equal(t, len(testCase.rangeToFilterLogsResponse), fakeLogClient.Count()) + } +} + +type logsInBlockRangeTestCase struct { + Label string + From int + To int + RangeToFilterLogsResponse map[string]filterLogsResponse + Logs []types.Log + FurthestBlockProcessed int +} + +func TestGetLogsInBlockRange(t *testing.T) { + from := 10 + to := 20 + testCases := []logsInBlockRangeTestCase{ + logsInBlockRangeTestCase{ + Label: "HAPPY_PATH", + From: from, + To: to, + RangeToFilterLogsResponse: map[string]filterLogsResponse{ + aRange(from, to): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub}, + FurthestBlockProcessed: to, + }, + logsInBlockRangeTestCase{ + Label: "SPLIT_REQUEST_BY_MAX_BLOCKS_IN_QUERY", + From: from, + To: from + maxBlocksInGetLogsQuery + 10, + RangeToFilterLogsResponse: map[string]filterLogsResponse{ + aRange(from, from+maxBlocksInGetLogsQuery-1): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + aRange(from+maxBlocksInGetLogsQuery, from+maxBlocksInGetLogsQuery+10): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{logStub, logStub}, + FurthestBlockProcessed: from + maxBlocksInGetLogsQuery + 10, + }, + logsInBlockRangeTestCase{ + Label: "SHORT_CIRCUIT_SEMAPHORE_BLOCKED_REQUESTS_ON_ERROR", + From: from, + To: from + (maxBlocksInGetLogsQuery * (getLogsRequestChunkSize + 1)), + RangeToFilterLogsResponse: map[string]filterLogsResponse{ + // Same number of responses as the getLogsRequestChunkSize since the + // error response will stop any further requests. + aRange(from, from+maxBlocksInGetLogsQuery-1): filterLogsResponse{ + Err: errUnexpected, + }, + aRange(from+maxBlocksInGetLogsQuery, from+(maxBlocksInGetLogsQuery*2)-1): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + aRange(from+(maxBlocksInGetLogsQuery*2), from+(maxBlocksInGetLogsQuery*3)-1): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + }, + Logs: []types.Log{}, + FurthestBlockProcessed: from - 1, + }, + logsInBlockRangeTestCase{ + Label: "CORRECT_FURTHEST_BLOCK_PROCESSED_ON_ERROR", + From: from, + To: from + maxBlocksInGetLogsQuery + 10, + RangeToFilterLogsResponse: map[string]filterLogsResponse{ + aRange(from, from+maxBlocksInGetLogsQuery-1): filterLogsResponse{ + Logs: []types.Log{ + logStub, + }, + }, + aRange(from+maxBlocksInGetLogsQuery, from+maxBlocksInGetLogsQuery+10): filterLogsResponse{ + Err: errUnexpected, + }}, + Logs: []types.Log{logStub}, + FurthestBlockProcessed: from + maxBlocksInGetLogsQuery - 1, + }, + } + + config.Store = &stubMiniHeaderStore{} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for _, testCase := range testCases { + fakeLogClient, err := newFakeLogClient(testCase.RangeToFilterLogsResponse) + require.NoError(t, err) + config.Client = fakeLogClient + watcher := New(config) + + logs, furthestBlockProcessed := watcher.getLogsInBlockRange(ctx, testCase.From, testCase.To) + require.Equal(t, testCase.FurthestBlockProcessed, furthestBlockProcessed, testCase.Label) + require.Equal(t, testCase.Logs, logs, testCase.Label) + assert.Equal(t, len(testCase.RangeToFilterLogsResponse), fakeLogClient.Count()) + } +} + +func aRange(from, to int) string { + r := fmt.Sprintf("%d-%d", from, to) + return r +} From 1a8bfcef384477a6af130e85cc06a9e89ea19351 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 15 Aug 2019 16:00:04 +0200 Subject: [PATCH 38/72] eth: roundswatcher with cache Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: eb28eb338ab84d4eae7611b4d530a0ea5be98797 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 151 +++++++++++++++++++++++++++++ eth/watchers/roundswatcher_test.go | 99 +++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 eth/watchers/roundswatcher.go create mode 100644 eth/watchers/roundswatcher_test.go diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go new file mode 100644 index 0000000..0029aee --- /dev/null +++ b/eth/watchers/roundswatcher.go @@ -0,0 +1,151 @@ +package watchers + +import ( + "fmt" + "log" + "math/big" + "sync" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/go-livepeer/eth/contracts" + + "github.com/golang/glog" + "github.com/livepeer/go-livepeer/eth" + "github.com/livepeer/go-livepeer/eth/blockwatch" +) + +// RoundsWatcher is a type for a thread safe in-memory cache that watches on-chain events for new initialized rounds +type RoundsWatcher struct { + mu sync.RWMutex + lastInitializedRound *big.Int + lastInitializedBlockHash [32]byte + + quit chan struct{} + + watcher BlockWatcher + lpEth eth.LivepeerEthClient + dec *EventDecoder +} + +// NewRoundsWatcher creates a new instance of RoundsWatcher and sets the initial cache through an RPC call to an ethereum node +func NewRoundsWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, lpEth eth.LivepeerEthClient) (*RoundsWatcher, error) { + dec, err := NewEventDecoder(roundsManagerAddr, contracts.RoundsManagerABI) + if err != nil { + return nil, err + } + lr, err := lpEth.LastInitializedRound() + if err != nil { + return nil, err + } + bh, err := lpEth.BlockHashForRound(lr) + if err != nil { + return nil, err + } + + return &RoundsWatcher{ + quit: make(chan struct{}), + watcher: watcher, + lpEth: lpEth, + lastInitializedRound: lr, + lastInitializedBlockHash: bh, + dec: dec, + }, nil +} + +// LastInitializedRound gets the last initialized round from cache +func (rw *RoundsWatcher) LastInitializedRound() *big.Int { + rw.mu.RLock() + defer rw.mu.RUnlock() + return rw.lastInitializedRound +} + +// LastInitializedBlockHash returns the blockhash of the block the last round was initiated in +func (rw *RoundsWatcher) LastInitializedBlockHash() [32]byte { + rw.mu.RLock() + defer rw.mu.RUnlock() + return rw.lastInitializedBlockHash +} + +func (rw *RoundsWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) { + rw.mu.Lock() + defer rw.mu.Unlock() + rw.lastInitializedRound = round + rw.lastInitializedBlockHash = hash +} + +// Watch the blockwatch subscription for NewRound events +func (rw *RoundsWatcher) Watch() { + lr, err := rw.lpEth.LastInitializedRound() + if err != nil { + glog.Errorf("error fetching initial lastInitializedRound value: %v", err) + } + + bh, err := rw.lpEth.BlockHashForRound(lr) + if err != nil { + glog.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) + } + rw.setLastInitializedRound(lr, bh) + + events := make(chan []*blockwatch.Event, 10) + sub := rw.watcher.Subscribe(events) + defer sub.Unsubscribe() + for { + select { + case <-rw.quit: + return + case err := <-sub.Err(): + log.Fatal(err) + case events := <-events: + rw.handleBlockEvents(events) + } + } +} + +// Stop watching for NewRound events +func (rw *RoundsWatcher) Stop() { + close(rw.quit) +} + +func (rw *RoundsWatcher) handleBlockEvents(events []*blockwatch.Event) { + for _, event := range events { + for _, log := range event.BlockHeader.Logs { + if event.Type == blockwatch.Removed { + log.Removed = true + } + if err := rw.handleLog(log); err != nil { + glog.Error(err) + } + } + } +} + +func (rw *RoundsWatcher) handleLog(log types.Log) error { + eventName, err := rw.dec.FindEventName(log) + if err != nil { + return fmt.Errorf("could not find event name: %v", err) + } + + if eventName != "NewRound" { + return fmt.Errorf("eventName is not NewRound") + } + + var nr contracts.RoundsManagerNewRound + if err := rw.dec.Decode("NewRound", log, &nr); err != nil { + return fmt.Errorf("unable to decode event: %v", err) + } + if log.Removed { + lr, err := rw.lpEth.LastInitializedRound() + if err != nil { + return err + } + bh, err := rw.lpEth.BlockHashForRound(lr) + if err != nil { + return err + } + rw.setLastInitializedRound(lr, bh) + } else { + rw.setLastInitializedRound(nr.Round, nr.BlockHash) + } + return nil +} diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go new file mode 100644 index 0000000..78dce2a --- /dev/null +++ b/eth/watchers/roundswatcher_test.go @@ -0,0 +1,99 @@ +package watchers + +import ( + "math/big" + "testing" + "time" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/go-livepeer/eth" + "github.com/livepeer/go-livepeer/eth/blockwatch" + "github.com/livepeer/go-livepeer/pm" + "github.com/stretchr/testify/assert" +) + +func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { + assert := assert.New(t) + rw := &RoundsWatcher{} + round := big.NewInt(5) + var hash [32]byte + copy(hash[:], "hello world") + rw.setLastInitializedRound(round, hash) + assert.Equal(rw.lastInitializedRound, round) + assert.Equal(rw.lastInitializedBlockHash, hash) + + r := rw.LastInitializedRound() + assert.Equal(r, round) + h := rw.LastInitializedBlockHash() + assert.Equal(h, hash) +} + +func TestWatchAndStop(t *testing.T) { + assert := assert.New(t) + lpEth := ð.StubClient{} + watcher := &stubBlockWatcher{} + rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(err) + + header := defaultMiniHeader() + newRoundEvent := newStubNewRoundLog() + + header.Logs = append(header.Logs, newRoundEvent) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + go rw.Watch() + time.Sleep(2 * time.Millisecond) + + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound := rw.LastInitializedRound() + assert.Zero(lastRound.Cmp(big.NewInt(8))) + bhForRound := rw.LastInitializedBlockHash() + var expectedHashForRound [32]byte + copy(expectedHashForRound[:], newRoundEvent.Data[:]) + assert.Equal(bhForRound, expectedHashForRound) + + // Test no NewRound events, values on rw remain the same + blockEvent.BlockHeader.Logs = header.Logs[:1] + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound = rw.LastInitializedRound() + assert.Zero(lastRound.Cmp(big.NewInt(8))) + bhForRound = rw.LastInitializedBlockHash() + copy(expectedHashForRound[:], newRoundEvent.Data[:]) + assert.Equal(bhForRound, expectedHashForRound) + + // Test RPC paths (event removed) + blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) + blockEvent.Type = blockwatch.Removed + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound = rw.LastInitializedRound() + bhForRound = rw.LastInitializedBlockHash() + assert.Equal(lastRound.Int64(), int64(0)) + assert.Equal(bhForRound, [32]byte{}) + + // Test Stop + rw.Stop() + time.Sleep(2 * time.Millisecond) + assert.True(watcher.sub.unsubscribed) +} + +func defaultMiniHeader() *blockwatch.MiniHeader { + block := &blockwatch.MiniHeader{ + Number: big.NewInt(450), + Parent: pm.RandHash(), + Hash: pm.RandHash(), + } + log := types.Log{ + Topics: []ethcommon.Hash{pm.RandHash(), pm.RandHash()}, + Data: pm.RandBytes(32), + BlockHash: block.Hash, + } + block.Logs = []types.Log{log} + return block +} From 2b931383b7d5224169ff0e20f659236531a8c766 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 19 Aug 2019 14:41:23 +0200 Subject: [PATCH 39/72] cmd, eth: init roundswatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 84c13f8b82cf4103ea55e7088304e1ead78c6156 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 0029aee..efaa4e6 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -2,7 +2,6 @@ package watchers import ( "fmt" - "log" "math/big" "sync" @@ -32,24 +31,14 @@ type RoundsWatcher struct { func NewRoundsWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, lpEth eth.LivepeerEthClient) (*RoundsWatcher, error) { dec, err := NewEventDecoder(roundsManagerAddr, contracts.RoundsManagerABI) if err != nil { - return nil, err - } - lr, err := lpEth.LastInitializedRound() - if err != nil { - return nil, err - } - bh, err := lpEth.BlockHashForRound(lr) - if err != nil { - return nil, err + return nil, fmt.Errorf("error creating decoder: %v", err) } return &RoundsWatcher{ - quit: make(chan struct{}), - watcher: watcher, - lpEth: lpEth, - lastInitializedRound: lr, - lastInitializedBlockHash: bh, - dec: dec, + quit: make(chan struct{}), + watcher: watcher, + lpEth: lpEth, + dec: dec, }, nil } @@ -80,7 +69,6 @@ func (rw *RoundsWatcher) Watch() { if err != nil { glog.Errorf("error fetching initial lastInitializedRound value: %v", err) } - bh, err := rw.lpEth.BlockHashForRound(lr) if err != nil { glog.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) @@ -95,7 +83,7 @@ func (rw *RoundsWatcher) Watch() { case <-rw.quit: return case err := <-sub.Err(): - log.Fatal(err) + glog.Error(err) case events := <-events: rw.handleBlockEvents(events) } From 7397e4fc58a75bb21aae7696d4d82f63a4b8b005 Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Mon, 26 Aug 2019 16:41:36 -0400 Subject: [PATCH 40/72] eth/watchers: Return nil for unknown events in RoundsWatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 7fefbe878d4540f52c02177ae7b303dae96cf74c Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 3 ++- eth/watchers/roundswatcher_test.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index efaa4e6..20031bd 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -111,7 +111,8 @@ func (rw *RoundsWatcher) handleBlockEvents(events []*blockwatch.Event) { func (rw *RoundsWatcher) handleLog(log types.Log) error { eventName, err := rw.dec.FindEventName(log) if err != nil { - return fmt.Errorf("could not find event name: %v", err) + // Noop if we cannot find the event name + return nil } if eventName != "NewRound" { diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go index 78dce2a..9ead20c 100644 --- a/eth/watchers/roundswatcher_test.go +++ b/eth/watchers/roundswatcher_test.go @@ -11,6 +11,7 @@ import ( "github.com/livepeer/go-livepeer/eth/blockwatch" "github.com/livepeer/go-livepeer/pm" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { @@ -83,6 +84,24 @@ func TestWatchAndStop(t *testing.T) { assert.True(watcher.sub.unsubscribed) } +func TestRoundsWatcher_HandleLog(t *testing.T) { + lpEth := ð.StubClient{} + watcher := &stubBlockWatcher{} + rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) + require.Nil(t, err) + + assert := assert.New(t) + + // Test unknown event + log := newStubBaseLog() + log.Topics = []ethcommon.Hash{ethcommon.BytesToHash([]byte("foo"))} + + err = rw.handleLog(log) + assert.Nil(err) + assert.Nil(rw.LastInitializedRound()) + assert.Equal([32]byte{}, rw.LastInitializedBlockHash()) +} + func defaultMiniHeader() *blockwatch.MiniHeader { block := &blockwatch.MiniHeader{ Number: big.NewInt(450), From 40b9ea21d34c47329aa5fe2d7e893357a88656ee Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Wed, 21 Aug 2019 16:10:36 +0200 Subject: [PATCH 41/72] cmd, eth: exit when watchers error Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: e2e86f042034e50c378c5167f3bd36452b6c9da1 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 8 ++++---- eth/watchers/roundswatcher_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 20031bd..37a8651 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -64,14 +64,14 @@ func (rw *RoundsWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) } // Watch the blockwatch subscription for NewRound events -func (rw *RoundsWatcher) Watch() { +func (rw *RoundsWatcher) Watch() error { lr, err := rw.lpEth.LastInitializedRound() if err != nil { - glog.Errorf("error fetching initial lastInitializedRound value: %v", err) + return fmt.Errorf("error fetching initial lastInitializedRound value: %v", err) } bh, err := rw.lpEth.BlockHashForRound(lr) if err != nil { - glog.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) + return fmt.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) } rw.setLastInitializedRound(lr, bh) @@ -81,7 +81,7 @@ func (rw *RoundsWatcher) Watch() { for { select { case <-rw.quit: - return + return nil case err := <-sub.Err(): glog.Error(err) case events := <-events: diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go index 9ead20c..b6e0a3c 100644 --- a/eth/watchers/roundswatcher_test.go +++ b/eth/watchers/roundswatcher_test.go @@ -1,6 +1,7 @@ package watchers import ( + "errors" "math/big" "testing" "time" @@ -82,6 +83,16 @@ func TestWatchAndStop(t *testing.T) { rw.Stop() time.Sleep(2 * time.Millisecond) assert.True(watcher.sub.unsubscribed) + + // Test watch error when RPC calls fail + rw = &RoundsWatcher{ + lpEth: ð.StubClient{ + RoundsErr: errors.New("roundswatcher error"), + }, + } + err = rw.Watch() + assert.NotNil(err) + assert.Contains(err.Error(), "roundswatcher error") } func TestRoundsWatcher_HandleLog(t *testing.T) { From 5f98b76721b02e7df93e5058ea05a1ce5bd628b3 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Wed, 21 Aug 2019 02:42:39 +0200 Subject: [PATCH 42/72] cmd: init senderwatcher on pm.Sender Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: a0465f78ade5bc4ccbb6afda847d145df6d3dc04 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher_test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go index b6e0a3c..7af30ee 100644 --- a/eth/watchers/roundswatcher_test.go +++ b/eth/watchers/roundswatcher_test.go @@ -7,10 +7,8 @@ import ( "time" ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/livepeer/go-livepeer/eth" "github.com/livepeer/go-livepeer/eth/blockwatch" - "github.com/livepeer/go-livepeer/pm" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -112,18 +110,3 @@ func TestRoundsWatcher_HandleLog(t *testing.T) { assert.Nil(rw.LastInitializedRound()) assert.Equal([32]byte{}, rw.LastInitializedBlockHash()) } - -func defaultMiniHeader() *blockwatch.MiniHeader { - block := &blockwatch.MiniHeader{ - Number: big.NewInt(450), - Parent: pm.RandHash(), - Hash: pm.RandHash(), - } - log := types.Log{ - Topics: []ethcommon.Hash{pm.RandHash(), pm.RandHash()}, - Data: pm.RandBytes(32), - BlockHash: block.Hash, - } - block.Logs = []types.Log{log} - return block -} From ba6f919d9b5234e69001b77c669f2c8653161c71 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 29 Aug 2019 19:23:43 +0200 Subject: [PATCH 43/72] eth: add transcoderPoolSize to RoundsWatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 5cc2e35221b1fbf1d994d17d6e6a0a128b010d81 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 32 ++++++++++++++++++++++++++++++ eth/watchers/roundswatcher_test.go | 21 ++++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 37a8651..1c24b9c 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -19,6 +19,7 @@ type RoundsWatcher struct { mu sync.RWMutex lastInitializedRound *big.Int lastInitializedBlockHash [32]byte + transcoderPoolSize *big.Int quit chan struct{} @@ -63,6 +64,18 @@ func (rw *RoundsWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) rw.lastInitializedBlockHash = hash } +func (rw *RoundsWatcher) GetTranscoderPoolSize() *big.Int { + rw.mu.RLock() + defer rw.mu.RUnlock() + return rw.transcoderPoolSize +} + +func (rw *RoundsWatcher) setTranscoderPoolSize(size *big.Int) { + rw.mu.Lock() + defer rw.mu.Unlock() + rw.transcoderPoolSize = size +} + // Watch the blockwatch subscription for NewRound events func (rw *RoundsWatcher) Watch() error { lr, err := rw.lpEth.LastInitializedRound() @@ -75,6 +88,10 @@ func (rw *RoundsWatcher) Watch() error { } rw.setLastInitializedRound(lr, bh) + if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { + glog.Errorf("error fetching initial transcoderPoolSize: %v", err) + } + events := make(chan []*blockwatch.Event, 10) sub := rw.watcher.Subscribe(events) defer sub.Unsubscribe() @@ -136,5 +153,20 @@ func (rw *RoundsWatcher) handleLog(log types.Log) error { } else { rw.setLastInitializedRound(nr.Round, nr.BlockHash) } + + // Get the active transcoder pool size when we receive a NewRound event + if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { + return err + } + + return nil +} + +func (rw *RoundsWatcher) fetchAndSetTranscoderPoolSize() error { + size, err := rw.lpEth.GetTranscoderPoolSize() + if err != nil { + return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + } + rw.setTranscoderPoolSize(size) return nil } diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go index 7af30ee..663cece 100644 --- a/eth/watchers/roundswatcher_test.go +++ b/eth/watchers/roundswatcher_test.go @@ -29,9 +29,21 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { assert.Equal(h, hash) } -func TestWatchAndStop(t *testing.T) { +func TestSetAndGet_TranscoderPoolSize(t *testing.T) { assert := assert.New(t) - lpEth := ð.StubClient{} + rw := &RoundsWatcher{} + size := big.NewInt(50) + rw.setTranscoderPoolSize(size) + assert.Equal(size, rw.transcoderPoolSize) + assert.Equal(size, rw.GetTranscoderPoolSize()) +} + +func TestRoundsWatcher_WatchAndStop(t *testing.T) { + assert := assert.New(t) + size := big.NewInt(50) + lpEth := ð.StubClient{ + PoolSize: size, + } watcher := &stubBlockWatcher{} rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) @@ -48,6 +60,7 @@ func TestWatchAndStop(t *testing.T) { go rw.Watch() time.Sleep(2 * time.Millisecond) + // New Round event watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) lastRound := rw.LastInitializedRound() @@ -56,8 +69,10 @@ func TestWatchAndStop(t *testing.T) { var expectedHashForRound [32]byte copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) + assert.Equal(size, rw.GetTranscoderPoolSize()) // Test no NewRound events, values on rw remain the same + rw.setTranscoderPoolSize(big.NewInt(10)) blockEvent.BlockHeader.Logs = header.Logs[:1] watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) @@ -66,6 +81,7 @@ func TestWatchAndStop(t *testing.T) { bhForRound = rw.LastInitializedBlockHash() copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) + assert.Equal(big.NewInt(10), rw.GetTranscoderPoolSize()) // Test RPC paths (event removed) blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) @@ -76,6 +92,7 @@ func TestWatchAndStop(t *testing.T) { bhForRound = rw.LastInitializedBlockHash() assert.Equal(lastRound.Int64(), int64(0)) assert.Equal(bhForRound, [32]byte{}) + assert.Equal(size, rw.GetTranscoderPoolSize()) // Test Stop rw.Stop() From c7ccd728fb34f46aa914d3764456b6a182096707 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Tue, 3 Sep 2019 14:03:11 +0200 Subject: [PATCH 44/72] pm: updates to Broker and SenderManager interface Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 5b74602591c67645a422b5414d0fb052158248e4 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 1c24b9c..61e370b 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -89,7 +89,7 @@ func (rw *RoundsWatcher) Watch() error { rw.setLastInitializedRound(lr, bh) if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { - glog.Errorf("error fetching initial transcoderPoolSize: %v", err) + return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) } events := make(chan []*blockwatch.Event, 10) From 2ecc05796acb0a1cf18738f1c2d61779227a6b73 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Fri, 4 Oct 2019 06:01:01 +0200 Subject: [PATCH 45/72] cmd, eth: RoundsWatcher subscription for SenderWatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 30942f652de14e5ac1eea7a7ea7e0ff7c6cc7f7d Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index 61e370b..d75b319 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -7,6 +7,7 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" "github.com/livepeer/go-livepeer/eth/contracts" "github.com/golang/glog" @@ -23,6 +24,8 @@ type RoundsWatcher struct { quit chan struct{} + subscription chan<- types.Log + watcher BlockWatcher lpEth eth.LivepeerEthClient dec *EventDecoder @@ -107,8 +110,19 @@ func (rw *RoundsWatcher) Watch() error { } } +// Subscribe allows one to subscribe to the round events emitted by the Watcher. +// To unsubscribe, simply call `Unsubscribe` on the returned subscription. +// The sink channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. +func (rw *RoundsWatcher) Subscribe(sink chan<- types.Log) event.Subscription { + return rw.subScope.Track(rw.subFeed.Subscribe(sink)) +} + // Stop watching for NewRound events func (rw *RoundsWatcher) Stop() { + if rw.subscription != nil { + close(rw.subscription) + } close(rw.quit) } @@ -140,6 +154,9 @@ func (rw *RoundsWatcher) handleLog(log types.Log) error { if err := rw.dec.Decode("NewRound", log, &nr); err != nil { return fmt.Errorf("unable to decode event: %v", err) } + + rw.subFeed.Send(log) + if log.Removed { lr, err := rw.lpEth.LastInitializedRound() if err != nil { From 429edbea1d44217f09cd7fa1307af29b920d0c19 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Sat, 5 Oct 2019 04:08:45 +0200 Subject: [PATCH 46/72] eth/watchers: refactor RoundsWatcher Subscription mechanism Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 1799b33eafcb50fa48171dd5509a50d05fb8fab4 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go index d75b319..6384fad 100644 --- a/eth/watchers/roundswatcher.go +++ b/eth/watchers/roundswatcher.go @@ -24,7 +24,8 @@ type RoundsWatcher struct { quit chan struct{} - subscription chan<- types.Log + subFeed event.Feed + subScope event.SubscriptionScope // Subscription scope tracking current live listeners watcher BlockWatcher lpEth eth.LivepeerEthClient @@ -110,7 +111,7 @@ func (rw *RoundsWatcher) Watch() error { } } -// Subscribe allows one to subscribe to the round events emitted by the Watcher. +// Subscribe allows one to subscribe to the block events emitted by the Watcher. // To unsubscribe, simply call `Unsubscribe` on the returned subscription. // The sink channel should have ample buffer space to avoid blocking other subscribers. // Slow subscribers are not dropped. @@ -120,10 +121,8 @@ func (rw *RoundsWatcher) Subscribe(sink chan<- types.Log) event.Subscription { // Stop watching for NewRound events func (rw *RoundsWatcher) Stop() { - if rw.subscription != nil { - close(rw.subscription) - } close(rw.quit) + rw.subScope.Close() } func (rw *RoundsWatcher) handleBlockEvents(events []*blockwatch.Event) { From 3e86a18e02703e47f4be3a6d8dfa79d932ad5167 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 28 Oct 2019 23:49:07 +0100 Subject: [PATCH 47/72] eth/blockwatch: remove block header not found info statement Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 894b8db2cf0a98a4ed9175e6063dd0dce1114e7a Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 1 - 1 file changed, 1 deletion(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index d2a0a00..6cde647 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -164,7 +164,6 @@ func (w *Watcher) pollNextBlock() error { nextHeader, err := w.client.HeaderByNumber(nextBlockNumber) if err != nil { if err == ethereum.NotFound { - glog.Infof("block header not found blockNumber=%v", nextBlockNumber) return nil // Noop and wait next polling interval } return err From ba31a78a29524aab82b16195be9ef257fad5385c Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 4 Nov 2019 11:46:50 +0100 Subject: [PATCH 48/72] eth/blockwatch: BackfillStartBlock option Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 74fc8f4cf6996533392f36b2ab17df320a958849 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 42 +++++++++++----- eth/blockwatch/block_watcher_test.go | 74 ++++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 16 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 6cde647..9789ab0 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -45,6 +45,7 @@ type Config struct { Store MiniHeaderStore PollingInterval time.Duration StartBlockDepth rpc.BlockNumber + BackfillStartBlock *big.Int BlockRetentionLimit int WithLogs bool Topics []common.Hash @@ -57,6 +58,7 @@ type Config struct { type Watcher struct { blockRetentionLimit int startBlockDepth rpc.BlockNumber + backfillStartBlock *big.Int stack *Stack client Client blockFeed event.Feed @@ -77,6 +79,7 @@ func New(config Config) *Watcher { pollingInterval: config.PollingInterval, blockRetentionLimit: config.BlockRetentionLimit, startBlockDepth: config.StartBlockDepth, + backfillStartBlock: config.BackfillStartBlock, stack: stack, client: config.Client, withLogs: config.WithLogs, @@ -281,28 +284,43 @@ func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, error) { events := []*Event{} + var ( + startBlockNum int + latestRetainedBlockNum int + blocksElapsed int + ) + latestRetainedBlock, err := w.stack.Peek() if err != nil { return events, err } - // No blocks stored, nowhere to backfill to - if latestRetainedBlock == nil { - return events, nil - } + latestBlock, err := w.client.HeaderByNumber(nil) if err != nil { return events, err } - blocksElapsed := big.NewInt(0).Sub(latestBlock.Number, latestRetainedBlock.Number) - if blocksElapsed.Int64() == 0 { + latestBlockNum := int(latestBlock.Number.Int64()) + + if latestRetainedBlock != nil { + latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) + // Events for latestRetainedBlock already processed, start at latestRetainedBlock + 1 + startBlockNum = latestRetainedBlockNum + 1 + } else if w.backfillStartBlock != nil { + startBlockNum = int(w.backfillStartBlock.Int64()) + } else { return events, nil } - glog.Infof("Some blocks have elapsed since last boot. Backfilling block events (this can take a while)... blocksElapsed=%v", blocksElapsed.Int64()) - startBlockNum := int(latestRetainedBlock.Number.Int64() + 1) - endBlockNum := int(latestRetainedBlock.Number.Int64() + blocksElapsed.Int64()) + if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed == 0 { + return events, nil + } + + glog.Infof("Backfilling block events (this can take a while)...\n") + glog.Infof("Start block: %v End block: %v Blocks elapsed: %v\n", startBlockNum, startBlockNum+blocksElapsed, blocksElapsed) + + endBlockNum := startBlockNum + blocksElapsed logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, endBlockNum) - if int64(furthestBlockProcessed) > latestRetainedBlock.Number.Int64() { + if furthestBlockProcessed > latestRetainedBlockNum { // If we have processed blocks further then the latestRetainedBlock in the DB, we // want to remove all blocks from the DB and insert the furthestBlockProcessed // Doing so will cause the BlockWatcher to start from that furthestBlockProcessed. @@ -337,9 +355,9 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro for _, log := range logs { blockHeader, ok := hashToBlockHeader[log.BlockHash] if !ok { - // TODO(fabio): Find a way to include the parent hash for the block as well. + // TODO: Find a way to include the parent hash for the block as well. // It's currently not an issue to omit it since we don't use the parent hash - // when processing block events in OrderWatcher. + // when processing block events in event watcher services blockHeader = &MiniHeader{ Hash: log.BlockHash, Number: big.NewInt(0).SetUint64(log.BlockNumber), diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 4f11787..06e4311 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -214,6 +214,48 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { assert.Equal(t, big.NewInt(30), headers[0].Number) } +func TestGetMissedEventsToBackfill_BackfillStartBlock(t *testing.T) { + // Fixture will return block 30 as the tip of the chain + fakeClient, err := newFakeClient("testdata/fake_client_fast_sync_fixture.json") + require.NoError(t, err) + + store := &stubMiniHeaderStore{} + + config.Store = store + config.Client = fakeClient + config.BackfillStartBlock = big.NewInt(0) + watcher := New(config) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + events, err := watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 1) + + // Check that block 30 is now in the DB as it is the last processed block + headers, err := store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 1) + assert.Equal(t, big.NewInt(30), headers[0].Number) + + store = &stubMiniHeaderStore{} + config.Store = store + config.BackfillStartBlock = big.NewInt(5) + watcher = New(config) + events, err = watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 1) + + // Check that block 30 is now in the DB as it is the last processed block + headers, err = store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 1) + assert.Equal(t, big.NewInt(30), headers[0].Number) + + config.BackfillStartBlock = nil +} + func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { // Fixture will return block 5 as the tip of the chain fakeClient, err := newFakeClient("testdata/fake_client_basic_fixture.json") @@ -246,6 +288,30 @@ func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { assert.Equal(t, big.NewInt(5), headers[0].Number) } +func TestGetMissedEventsToBackfill_NOOP(t *testing.T) { + // No last retained block + // No config.BackfillStartBlock + + fakeClient, err := newFakeClient("testdata/fake_client_basic_fixture.json") + require.NoError(t, err) + + store := &stubMiniHeaderStore{} + + config.Store = store + config.Client = fakeClient + watcher := New(config) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + events, err := watcher.getMissedEventsToBackfill(ctx) + require.NoError(t, err) + assert.Len(t, events, 0) + + headers, err := store.FindAllMiniHeadersSortedByNumber() + require.NoError(t, err) + require.Len(t, headers, 0) +} + var logStub = types.Log{ Address: common.HexToAddress("0x21ab6c9fac80c59d401b37cb43f81ea9dde7fe34"), Topics: []common.Hash{ @@ -399,7 +465,7 @@ func TestGetLogsInBlockRange(t *testing.T) { }, }, }, - Logs: []types.Log{logStub}, + Logs: []types.Log{logStub}, FurthestBlockProcessed: to, }, logsInBlockRangeTestCase{ @@ -418,7 +484,7 @@ func TestGetLogsInBlockRange(t *testing.T) { }, }, }, - Logs: []types.Log{logStub, logStub}, + Logs: []types.Log{logStub, logStub}, FurthestBlockProcessed: from + maxBlocksInGetLogsQuery + 10, }, logsInBlockRangeTestCase{ @@ -442,7 +508,7 @@ func TestGetLogsInBlockRange(t *testing.T) { }, }, }, - Logs: []types.Log{}, + Logs: []types.Log{}, FurthestBlockProcessed: from - 1, }, logsInBlockRangeTestCase{ @@ -458,7 +524,7 @@ func TestGetLogsInBlockRange(t *testing.T) { aRange(from+maxBlocksInGetLogsQuery, from+maxBlocksInGetLogsQuery+10): filterLogsResponse{ Err: errUnexpected, }}, - Logs: []types.Log{logStub}, + Logs: []types.Log{logStub}, FurthestBlockProcessed: from + maxBlocksInGetLogsQuery - 1, }, } From ee275cafe2027fec3ae655e2d6cf8aa0230cc4b3 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 4 Nov 2019 12:21:50 +0100 Subject: [PATCH 49/72] cmd: determine backfilling startblock based on currentround and last seen block Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 7d21c328e49e0afc2929c7dbb8d42e31ebd6de04 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 9789ab0..430b902 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -301,12 +301,12 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro } latestBlockNum := int(latestBlock.Number.Int64()) - if latestRetainedBlock != nil { + if w.backfillStartBlock != nil { + startBlockNum = int(w.backfillStartBlock.Int64()) + } else if latestRetainedBlock != nil { latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) // Events for latestRetainedBlock already processed, start at latestRetainedBlock + 1 startBlockNum = latestRetainedBlockNum + 1 - } else if w.backfillStartBlock != nil { - startBlockNum = int(w.backfillStartBlock.Int64()) } else { return events, nil } @@ -318,8 +318,7 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro glog.Infof("Backfilling block events (this can take a while)...\n") glog.Infof("Start block: %v End block: %v Blocks elapsed: %v\n", startBlockNum, startBlockNum+blocksElapsed, blocksElapsed) - endBlockNum := startBlockNum + blocksElapsed - logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, endBlockNum) + logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, latestBlockNum) if furthestBlockProcessed > latestRetainedBlockNum { // If we have processed blocks further then the latestRetainedBlock in the DB, we // want to remove all blocks from the DB and insert the furthestBlockProcessed From 6b750f6c0654dfef8322960716592049db22a2b8 Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Fri, 15 Nov 2019 10:22:53 -0500 Subject: [PATCH 50/72] blockwatch: Log Watcher error Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 2fb674c587760f121695bffb9415fa99b89ec85d Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 430b902..0412197 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -121,7 +121,7 @@ func (w *Watcher) Watch(ctx context.Context) error { return nil case <-ticker.C: if err := w.pollNextBlock(); err != nil { - glog.Error("blockwatch.Watcher error encountered") + glog.Errorf("blockwatch.Watcher error encountered: %v", err) } } } From 1662175067cce3b7051884bac27489555e639116 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Fri, 24 Jan 2020 15:29:31 +0100 Subject: [PATCH 51/72] eth/blockwatch: reduce logging level of recoverable errors and change their language to be perceived as not interfering with node operations Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: d857c1b926b38b2a749aa413e382a8abca1bc91d Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 0412197..955b6b5 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -121,7 +121,7 @@ func (w *Watcher) Watch(ctx context.Context) error { return nil case <-ticker.C: if err := w.pollNextBlock(); err != nil { - glog.Errorf("blockwatch.Watcher error encountered: %v", err) + glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%v", err) } } } @@ -199,7 +199,7 @@ func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ( // returned. This is expected to happen sometimes, and we simply return the events gathered so // far and pick back up where we left off on the next polling interval. if isUnknownBlockErr(err) { - glog.Infof("failed to get logs for block nextHeader=%v", nextHeader) + glog.V(5).Infof("missing logs for blockNumber=%v - fetching on next polling interval", nextHeader.Number) return events, nil } return events, err @@ -227,7 +227,7 @@ func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ( nextParentHeader, err := w.client.HeaderByHash(nextHeader.Parent) if err != nil { if err == ethereum.NotFound { - glog.Infof("block header not found blockHash=%v", nextHeader.Parent.Hex()) + glog.V(5).Infof("block header not found blockHash=%v - fetching on next polling interval", nextHeader.Parent.Hex()) // Noop and wait next polling interval. We remove the popped blocks // and refetch them on the next polling interval. return events, nil @@ -245,7 +245,7 @@ func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ( // returned. This is expected to happen sometimes, and we simply return the events gathered so // far and pick back up where we left off on the next polling interval. if isUnknownBlockErr(err) { - glog.Infof("failed to get logs for block nextHeader=%v", nextHeader) + glog.V(5).Infof("missing logs for blockNumber=%v - fetching on next polling interval", nextHeader.Number) return events, nil } return events, err From e8efe106db9e738684b10f8c062c156b0aebedab Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 6 Feb 2020 03:31:58 +0100 Subject: [PATCH 52/72] eth/watchers: extend roundsWatcher to be a more generic timeWatcher that watchers for both latest rounds and latest blocks Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: b4605f705bcad5e355bb0c738d784e4503089900 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/roundswatcher.go | 188 ----------------------- eth/watchers/roundswatcher_test.go | 129 ---------------- eth/watchers/timewatcher.go | 233 +++++++++++++++++++++++++++++ eth/watchers/timewatcher_test.go | 215 ++++++++++++++++++++++++++ 4 files changed, 448 insertions(+), 317 deletions(-) delete mode 100644 eth/watchers/roundswatcher.go delete mode 100644 eth/watchers/roundswatcher_test.go create mode 100644 eth/watchers/timewatcher.go create mode 100644 eth/watchers/timewatcher_test.go diff --git a/eth/watchers/roundswatcher.go b/eth/watchers/roundswatcher.go deleted file mode 100644 index 6384fad..0000000 --- a/eth/watchers/roundswatcher.go +++ /dev/null @@ -1,188 +0,0 @@ -package watchers - -import ( - "fmt" - "math/big" - "sync" - - ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/event" - "github.com/livepeer/go-livepeer/eth/contracts" - - "github.com/golang/glog" - "github.com/livepeer/go-livepeer/eth" - "github.com/livepeer/go-livepeer/eth/blockwatch" -) - -// RoundsWatcher is a type for a thread safe in-memory cache that watches on-chain events for new initialized rounds -type RoundsWatcher struct { - mu sync.RWMutex - lastInitializedRound *big.Int - lastInitializedBlockHash [32]byte - transcoderPoolSize *big.Int - - quit chan struct{} - - subFeed event.Feed - subScope event.SubscriptionScope // Subscription scope tracking current live listeners - - watcher BlockWatcher - lpEth eth.LivepeerEthClient - dec *EventDecoder -} - -// NewRoundsWatcher creates a new instance of RoundsWatcher and sets the initial cache through an RPC call to an ethereum node -func NewRoundsWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, lpEth eth.LivepeerEthClient) (*RoundsWatcher, error) { - dec, err := NewEventDecoder(roundsManagerAddr, contracts.RoundsManagerABI) - if err != nil { - return nil, fmt.Errorf("error creating decoder: %v", err) - } - - return &RoundsWatcher{ - quit: make(chan struct{}), - watcher: watcher, - lpEth: lpEth, - dec: dec, - }, nil -} - -// LastInitializedRound gets the last initialized round from cache -func (rw *RoundsWatcher) LastInitializedRound() *big.Int { - rw.mu.RLock() - defer rw.mu.RUnlock() - return rw.lastInitializedRound -} - -// LastInitializedBlockHash returns the blockhash of the block the last round was initiated in -func (rw *RoundsWatcher) LastInitializedBlockHash() [32]byte { - rw.mu.RLock() - defer rw.mu.RUnlock() - return rw.lastInitializedBlockHash -} - -func (rw *RoundsWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) { - rw.mu.Lock() - defer rw.mu.Unlock() - rw.lastInitializedRound = round - rw.lastInitializedBlockHash = hash -} - -func (rw *RoundsWatcher) GetTranscoderPoolSize() *big.Int { - rw.mu.RLock() - defer rw.mu.RUnlock() - return rw.transcoderPoolSize -} - -func (rw *RoundsWatcher) setTranscoderPoolSize(size *big.Int) { - rw.mu.Lock() - defer rw.mu.Unlock() - rw.transcoderPoolSize = size -} - -// Watch the blockwatch subscription for NewRound events -func (rw *RoundsWatcher) Watch() error { - lr, err := rw.lpEth.LastInitializedRound() - if err != nil { - return fmt.Errorf("error fetching initial lastInitializedRound value: %v", err) - } - bh, err := rw.lpEth.BlockHashForRound(lr) - if err != nil { - return fmt.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) - } - rw.setLastInitializedRound(lr, bh) - - if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) - } - - events := make(chan []*blockwatch.Event, 10) - sub := rw.watcher.Subscribe(events) - defer sub.Unsubscribe() - for { - select { - case <-rw.quit: - return nil - case err := <-sub.Err(): - glog.Error(err) - case events := <-events: - rw.handleBlockEvents(events) - } - } -} - -// Subscribe allows one to subscribe to the block events emitted by the Watcher. -// To unsubscribe, simply call `Unsubscribe` on the returned subscription. -// The sink channel should have ample buffer space to avoid blocking other subscribers. -// Slow subscribers are not dropped. -func (rw *RoundsWatcher) Subscribe(sink chan<- types.Log) event.Subscription { - return rw.subScope.Track(rw.subFeed.Subscribe(sink)) -} - -// Stop watching for NewRound events -func (rw *RoundsWatcher) Stop() { - close(rw.quit) - rw.subScope.Close() -} - -func (rw *RoundsWatcher) handleBlockEvents(events []*blockwatch.Event) { - for _, event := range events { - for _, log := range event.BlockHeader.Logs { - if event.Type == blockwatch.Removed { - log.Removed = true - } - if err := rw.handleLog(log); err != nil { - glog.Error(err) - } - } - } -} - -func (rw *RoundsWatcher) handleLog(log types.Log) error { - eventName, err := rw.dec.FindEventName(log) - if err != nil { - // Noop if we cannot find the event name - return nil - } - - if eventName != "NewRound" { - return fmt.Errorf("eventName is not NewRound") - } - - var nr contracts.RoundsManagerNewRound - if err := rw.dec.Decode("NewRound", log, &nr); err != nil { - return fmt.Errorf("unable to decode event: %v", err) - } - - rw.subFeed.Send(log) - - if log.Removed { - lr, err := rw.lpEth.LastInitializedRound() - if err != nil { - return err - } - bh, err := rw.lpEth.BlockHashForRound(lr) - if err != nil { - return err - } - rw.setLastInitializedRound(lr, bh) - } else { - rw.setLastInitializedRound(nr.Round, nr.BlockHash) - } - - // Get the active transcoder pool size when we receive a NewRound event - if err := rw.fetchAndSetTranscoderPoolSize(); err != nil { - return err - } - - return nil -} - -func (rw *RoundsWatcher) fetchAndSetTranscoderPoolSize() error { - size, err := rw.lpEth.GetTranscoderPoolSize() - if err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) - } - rw.setTranscoderPoolSize(size) - return nil -} diff --git a/eth/watchers/roundswatcher_test.go b/eth/watchers/roundswatcher_test.go deleted file mode 100644 index 663cece..0000000 --- a/eth/watchers/roundswatcher_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package watchers - -import ( - "errors" - "math/big" - "testing" - "time" - - ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/livepeer/go-livepeer/eth" - "github.com/livepeer/go-livepeer/eth/blockwatch" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { - assert := assert.New(t) - rw := &RoundsWatcher{} - round := big.NewInt(5) - var hash [32]byte - copy(hash[:], "hello world") - rw.setLastInitializedRound(round, hash) - assert.Equal(rw.lastInitializedRound, round) - assert.Equal(rw.lastInitializedBlockHash, hash) - - r := rw.LastInitializedRound() - assert.Equal(r, round) - h := rw.LastInitializedBlockHash() - assert.Equal(h, hash) -} - -func TestSetAndGet_TranscoderPoolSize(t *testing.T) { - assert := assert.New(t) - rw := &RoundsWatcher{} - size := big.NewInt(50) - rw.setTranscoderPoolSize(size) - assert.Equal(size, rw.transcoderPoolSize) - assert.Equal(size, rw.GetTranscoderPoolSize()) -} - -func TestRoundsWatcher_WatchAndStop(t *testing.T) { - assert := assert.New(t) - size := big.NewInt(50) - lpEth := ð.StubClient{ - PoolSize: size, - } - watcher := &stubBlockWatcher{} - rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) - assert.Nil(err) - - header := defaultMiniHeader() - newRoundEvent := newStubNewRoundLog() - - header.Logs = append(header.Logs, newRoundEvent) - blockEvent := &blockwatch.Event{ - Type: blockwatch.Added, - BlockHeader: header, - } - - go rw.Watch() - time.Sleep(2 * time.Millisecond) - - // New Round event - watcher.sink <- []*blockwatch.Event{blockEvent} - time.Sleep(2 * time.Millisecond) - lastRound := rw.LastInitializedRound() - assert.Zero(lastRound.Cmp(big.NewInt(8))) - bhForRound := rw.LastInitializedBlockHash() - var expectedHashForRound [32]byte - copy(expectedHashForRound[:], newRoundEvent.Data[:]) - assert.Equal(bhForRound, expectedHashForRound) - assert.Equal(size, rw.GetTranscoderPoolSize()) - - // Test no NewRound events, values on rw remain the same - rw.setTranscoderPoolSize(big.NewInt(10)) - blockEvent.BlockHeader.Logs = header.Logs[:1] - watcher.sink <- []*blockwatch.Event{blockEvent} - time.Sleep(2 * time.Millisecond) - lastRound = rw.LastInitializedRound() - assert.Zero(lastRound.Cmp(big.NewInt(8))) - bhForRound = rw.LastInitializedBlockHash() - copy(expectedHashForRound[:], newRoundEvent.Data[:]) - assert.Equal(bhForRound, expectedHashForRound) - assert.Equal(big.NewInt(10), rw.GetTranscoderPoolSize()) - - // Test RPC paths (event removed) - blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) - blockEvent.Type = blockwatch.Removed - watcher.sink <- []*blockwatch.Event{blockEvent} - time.Sleep(2 * time.Millisecond) - lastRound = rw.LastInitializedRound() - bhForRound = rw.LastInitializedBlockHash() - assert.Equal(lastRound.Int64(), int64(0)) - assert.Equal(bhForRound, [32]byte{}) - assert.Equal(size, rw.GetTranscoderPoolSize()) - - // Test Stop - rw.Stop() - time.Sleep(2 * time.Millisecond) - assert.True(watcher.sub.unsubscribed) - - // Test watch error when RPC calls fail - rw = &RoundsWatcher{ - lpEth: ð.StubClient{ - RoundsErr: errors.New("roundswatcher error"), - }, - } - err = rw.Watch() - assert.NotNil(err) - assert.Contains(err.Error(), "roundswatcher error") -} - -func TestRoundsWatcher_HandleLog(t *testing.T) { - lpEth := ð.StubClient{} - watcher := &stubBlockWatcher{} - rw, err := NewRoundsWatcher(stubRoundsManagerAddr, watcher, lpEth) - require.Nil(t, err) - - assert := assert.New(t) - - // Test unknown event - log := newStubBaseLog() - log.Topics = []ethcommon.Hash{ethcommon.BytesToHash([]byte("foo"))} - - err = rw.handleLog(log) - assert.Nil(err) - assert.Nil(rw.LastInitializedRound()) - assert.Equal([32]byte{}, rw.LastInitializedBlockHash()) -} diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go new file mode 100644 index 0000000..e61dd23 --- /dev/null +++ b/eth/watchers/timewatcher.go @@ -0,0 +1,233 @@ +package watchers + +import ( + "fmt" + "math/big" + "sync" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/livepeer/go-livepeer/eth/contracts" + + "github.com/golang/glog" + "github.com/livepeer/go-livepeer/eth" + "github.com/livepeer/go-livepeer/eth/blockwatch" +) + +// TimeWatcher is a type for a thread safe in-memory cache that watches for the following on-chain events: +// * New rounds +// * New blocks +// * Transcoder (de)activation + +// TimeWatcher allows for subscriptions to certain data feeds using a caller provided sink channel +// consumers of the TimeWatcher can subscribe to following data feeds: +// * Last Initialized Round Number +// * Last Seen Block Number +type TimeWatcher struct { + // state + mu sync.RWMutex + lastInitializedRound *big.Int + lastInitializedBlockHash [32]byte + transcoderPoolSize *big.Int + lastSeenBlock *big.Int + + // last initialized round number subscription feeds + roundSubFeed event.Feed + roundSubScope event.SubscriptionScope + // last seen block number subscription feeds + blockSubFeed event.Feed + blockSubScope event.SubscriptionScope + + watcher BlockWatcher + lpEth eth.LivepeerEthClient + dec *EventDecoder + + quit chan struct{} +} + +// NewTimeWatcher creates a new instance of TimeWatcher and sets the initial cache through an RPC call to an ethereum node +func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, lpEth eth.LivepeerEthClient) (*TimeWatcher, error) { + dec, err := NewEventDecoder(roundsManagerAddr, contracts.RoundsManagerABI) + if err != nil { + return nil, fmt.Errorf("error creating decoder: %v", err) + } + + return &TimeWatcher{ + quit: make(chan struct{}), + watcher: watcher, + lpEth: lpEth, + dec: dec, + }, nil +} + +// LastInitializedRound gets the last initialized round from cache +func (tw *TimeWatcher) LastInitializedRound() *big.Int { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.lastInitializedRound +} + +// LastInitializedBlockHash returns the blockhash of the block the last round was initiated in +func (tw *TimeWatcher) LastInitializedBlockHash() [32]byte { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.lastInitializedBlockHash +} + +func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) { + tw.mu.Lock() + defer tw.mu.Unlock() + tw.lastInitializedRound = round + tw.lastInitializedBlockHash = hash +} + +func (tw *TimeWatcher) GetTranscoderPoolSize() *big.Int { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.transcoderPoolSize +} + +func (tw *TimeWatcher) setTranscoderPoolSize(size *big.Int) { + tw.mu.Lock() + defer tw.mu.Unlock() + tw.transcoderPoolSize = size +} + +func (tw *TimeWatcher) LastSeenBlock() *big.Int { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.lastSeenBlock +} + +func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { + tw.mu.Lock() + defer tw.mu.Unlock() + tw.lastSeenBlock = blockNum +} + +// Watch the blockwatch subscription for NewRound events +func (tw *TimeWatcher) Watch() error { + lr, err := tw.lpEth.LastInitializedRound() + if err != nil { + return fmt.Errorf("error fetching initial lastInitializedRound value: %v", err) + } + bh, err := tw.lpEth.BlockHashForRound(lr) + if err != nil { + return fmt.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) + } + tw.setLastInitializedRound(lr, bh) + + if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { + return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + } + + events := make(chan []*blockwatch.Event, 10) + sub := tw.watcher.Subscribe(events) + defer sub.Unsubscribe() + for { + select { + case <-tw.quit: + return nil + case err := <-sub.Err(): + glog.Error(err) + case events := <-events: + tw.handleBlockEvents(events) + } + } +} + +// SubscribeRounds allows one to subscribe to new round events +// To unsubscribe, simply call `Unsubscribe` on the returned subscription. +// The sink channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. +func (tw *TimeWatcher) SubscribeRounds(sink chan<- types.Log) event.Subscription { + return tw.roundSubScope.Track(tw.roundSubFeed.Subscribe(sink)) +} + +// SubscribeBlocks allows one to subscribe to newly seen block numbers +// To unsubscribe, simply call `Unsubscribe` on the returned subscription. +// The sink channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. +func (tw *TimeWatcher) SubscribeBlocks(sink chan<- *big.Int) event.Subscription { + return tw.blockSubScope.Track(tw.blockSubFeed.Subscribe(sink)) +} + +// Stop TimeWatcher +func (tw *TimeWatcher) Stop() { + close(tw.quit) + tw.blockSubScope.Close() + tw.roundSubScope.Close() +} + +func (tw *TimeWatcher) handleBlockEvents(events []*blockwatch.Event) { + for _, event := range events { + tw.handleBlockNum(event) + for _, log := range event.BlockHeader.Logs { + if event.Type == blockwatch.Removed { + log.Removed = true + } + if err := tw.handleLog(log); err != nil { + glog.Error(err) + } + } + } +} + +func (tw *TimeWatcher) handleBlockNum(event *blockwatch.Event) { + last := tw.LastSeenBlock() + new := event.BlockHeader.Number + if last == nil || last.Cmp(new) != 0 { + tw.setLastSeenBlock(new) + tw.blockSubFeed.Send(new) + } +} + +func (tw *TimeWatcher) handleLog(log types.Log) error { + eventName, err := tw.dec.FindEventName(log) + if err != nil { + // Noop if we cannot find the event name + return nil + } + + if eventName != "NewRound" { + return fmt.Errorf("eventName is not NewRound") + } + + var nr contracts.RoundsManagerNewRound + if err := tw.dec.Decode("NewRound", log, &nr); err != nil { + return fmt.Errorf("unable to decode event: %v", err) + } + + tw.roundSubFeed.Send(log) + + if log.Removed { + lr, err := tw.lpEth.LastInitializedRound() + if err != nil { + return err + } + bh, err := tw.lpEth.BlockHashForRound(lr) + if err != nil { + return err + } + tw.setLastInitializedRound(lr, bh) + } else { + tw.setLastInitializedRound(nr.Round, nr.BlockHash) + } + + // Get the active transcoder pool size when we receive a NewRound event + if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { + return err + } + + return nil +} + +func (tw *TimeWatcher) fetchAndSetTranscoderPoolSize() error { + size, err := tw.lpEth.GetTranscoderPoolSize() + if err != nil { + return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + } + tw.setTranscoderPoolSize(size) + return nil +} diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go new file mode 100644 index 0000000..4025dce --- /dev/null +++ b/eth/watchers/timewatcher_test.go @@ -0,0 +1,215 @@ +package watchers + +import ( + "errors" + "math/big" + "testing" + "time" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/go-livepeer/eth" + "github.com/livepeer/go-livepeer/eth/blockwatch" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { + assert := assert.New(t) + tw := &TimeWatcher{} + round := big.NewInt(5) + var hash [32]byte + copy(hash[:], "hello world") + tw.setLastInitializedRound(round, hash) + assert.Equal(tw.lastInitializedRound, round) + assert.Equal(tw.lastInitializedBlockHash, hash) + + r := tw.LastInitializedRound() + assert.Equal(r, round) + h := tw.LastInitializedBlockHash() + assert.Equal(h, hash) +} + +func TestSetAndGet_TranscoderPoolSize(t *testing.T) { + assert := assert.New(t) + tw := &TimeWatcher{} + size := big.NewInt(50) + tw.setTranscoderPoolSize(size) + assert.Equal(size, tw.transcoderPoolSize) + assert.Equal(size, tw.GetTranscoderPoolSize()) +} + +func TestRoundsWatcher_WatchAndStop(t *testing.T) { + assert := assert.New(t) + size := big.NewInt(50) + lpEth := ð.StubClient{ + PoolSize: size, + } + watcher := &stubBlockWatcher{} + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(err) + + header := defaultMiniHeader() + newRoundEvent := newStubNewRoundLog() + + header.Logs = append(header.Logs, newRoundEvent) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + go tw.Watch() + time.Sleep(2 * time.Millisecond) + + // New Round event + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound := tw.LastInitializedRound() + assert.Zero(lastRound.Cmp(big.NewInt(8))) + bhForRound := tw.LastInitializedBlockHash() + var expectedHashForRound [32]byte + copy(expectedHashForRound[:], newRoundEvent.Data[:]) + assert.Equal(bhForRound, expectedHashForRound) + assert.Equal(size, tw.GetTranscoderPoolSize()) + + // Test no NewRound events, values on rw remain the same + tw.setTranscoderPoolSize(big.NewInt(10)) + blockEvent.BlockHeader.Logs = header.Logs[:1] + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound = tw.LastInitializedRound() + assert.Zero(lastRound.Cmp(big.NewInt(8))) + bhForRound = tw.LastInitializedBlockHash() + copy(expectedHashForRound[:], newRoundEvent.Data[:]) + assert.Equal(bhForRound, expectedHashForRound) + assert.Equal(big.NewInt(10), tw.GetTranscoderPoolSize()) + + // Test RPC paths (event removed) + blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) + blockEvent.Type = blockwatch.Removed + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + lastRound = tw.LastInitializedRound() + bhForRound = tw.LastInitializedBlockHash() + assert.Equal(lastRound.Int64(), int64(0)) + assert.Equal(bhForRound, [32]byte{}) + assert.Equal(size, tw.GetTranscoderPoolSize()) + + // Test Stop + tw.Stop() + time.Sleep(2 * time.Millisecond) + assert.True(watcher.sub.unsubscribed) + + // Test watch error when RPC calls fail + tw = &TimeWatcher{ + lpEth: ð.StubClient{ + RoundsErr: errors.New("roundswatcher error"), + }, + } + err = tw.Watch() + assert.NotNil(err) + assert.Contains(err.Error(), "roundswatcher error") +} + +func TestRoundsWatcher_HandleLog(t *testing.T) { + lpEth := ð.StubClient{} + watcher := &stubBlockWatcher{} + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + require.Nil(t, err) + + assert := assert.New(t) + + // Test unknown event + log := newStubBaseLog() + log.Topics = []ethcommon.Hash{ethcommon.BytesToHash([]byte("foo"))} + + err = tw.handleLog(log) + assert.Nil(err) + assert.Nil(tw.LastInitializedRound()) + assert.Equal([32]byte{}, tw.LastInitializedBlockHash()) +} + +func TestLastSeenBlock(t *testing.T) { + assert := assert.New(t) + tw := &TimeWatcher{} + block := big.NewInt(5) + + tw.setLastSeenBlock(block) + tw.LastSeenBlock() + assert.Equal(big.NewInt(5), tw.LastSeenBlock()) +} + +func TestHandleBlockNum(t *testing.T) { + assert := assert.New(t) + watcher := &stubBlockWatcher{} + + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + assert.Nil(err) + header := defaultMiniHeader() + header.Number = big.NewInt(10) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + go tw.Watch() + time.Sleep(2 * time.Millisecond) + + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + assert.Equal(tw.LastSeenBlock(), header.Number) +} + +func TestSubscribeBlocks(t *testing.T) { + assert := assert.New(t) + watcher := &stubBlockWatcher{} + + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + assert.Nil(err) + header := defaultMiniHeader() + header.Number = big.NewInt(10) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + events := make(chan *big.Int, 10) + sub := tw.SubscribeBlocks(events) + defer sub.Unsubscribe() + + go tw.Watch() + time.Sleep(2 * time.Millisecond) + + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + update := <-events + assert.Equal(update, header.Number) +} + +func TestSubscribeRounds(t *testing.T) { + assert := assert.New(t) + watcher := &stubBlockWatcher{} + + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + assert.Nil(err) + header := defaultMiniHeader() + newRoundEvent := newStubNewRoundLog() + + header.Logs = append(header.Logs, newRoundEvent) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + events := make(chan types.Log, 10) + sub := tw.SubscribeRounds(events) + defer sub.Unsubscribe() + + go tw.Watch() + time.Sleep(2 * time.Millisecond) + + watcher.sink <- []*blockwatch.Event{blockEvent} + time.Sleep(2 * time.Millisecond) + update := <-events + assert.Equal(newRoundEvent, update) +} From 0d17715bc28dfc656736329a5ef628d2a7dc38c2 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Tue, 25 Feb 2020 18:33:35 +0100 Subject: [PATCH 53/72] eth/watcher: cache last seen block when starting timeWatcher Watch() loop Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 466185d0f90a1ed6179db8d9ea25834a5f9ec0a5 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index e61dd23..002ea37 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -110,16 +110,16 @@ func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { func (tw *TimeWatcher) Watch() error { lr, err := tw.lpEth.LastInitializedRound() if err != nil { - return fmt.Errorf("error fetching initial lastInitializedRound value: %v", err) + return fmt.Errorf("error fetching initial lastInitializedRound value err=%v", err) } bh, err := tw.lpEth.BlockHashForRound(lr) if err != nil { - return fmt.Errorf("error fetching initial lastInitializedBlockHash value: %v", err) + return fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) } tw.setLastInitializedRound(lr, bh) if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + return fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) } events := make(chan []*blockwatch.Event, 10) From 67c22d725d24d4cbdee6afa74c94b6aef8e38a8d Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 16 Mar 2020 18:02:09 +0100 Subject: [PATCH 54/72] eth: prefill last seen block state in timewatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 06cdd9176cb4854d4134d9b424647556fa3b4045 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 10 ++++++ eth/watchers/timewatcher_test.go | 61 +++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 12 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 002ea37..10f4a38 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -122,6 +122,16 @@ func (tw *TimeWatcher) Watch() error { return fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) } + lastSeenBlock, err := tw.watcher.GetLatestBlock() + if err != nil { + return fmt.Errorf("error fetching last seen block err=%v", err) + } + blockNum := big.NewInt(0) + if lastSeenBlock != nil { + blockNum = lastSeenBlock.Number + } + tw.setLastSeenBlock(blockNum) + events := make(chan []*blockwatch.Event, 10) sub := tw.watcher.Subscribe(events) defer sub.Unsubscribe() diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index 4025dce..ea4a2cc 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/livepeer/go-livepeer/eth" @@ -39,13 +40,21 @@ func TestSetAndGet_TranscoderPoolSize(t *testing.T) { assert.Equal(size, tw.GetTranscoderPoolSize()) } -func TestRoundsWatcher_WatchAndStop(t *testing.T) { +func TestTimeWatcher_WatchAndStop(t *testing.T) { assert := assert.New(t) size := big.NewInt(50) + block := big.NewInt(10) + round := big.NewInt(1) + hash := ethcommon.HexToHash("foo") lpEth := ð.StubClient{ - PoolSize: size, + PoolSize: size, + BlockNum: block, + BlockHashToReturn: hash, + Round: round, + } + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: block}, } - watcher := &stubBlockWatcher{} tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) @@ -60,6 +69,26 @@ func TestRoundsWatcher_WatchAndStop(t *testing.T) { go tw.Watch() time.Sleep(2 * time.Millisecond) + // Check state initialization + bh := tw.LastInitializedBlockHash() + assert.Equal(hash, common.BytesToHash(bh[:])) + assert.Equal(round, tw.LastInitializedRound()) + assert.Equal(size, tw.GetTranscoderPoolSize()) + assert.Equal(block, tw.LastSeenBlock()) + tw.Stop() + + // if watcher.GetLatestBlock() == nil, initialise lastSeenBlock to big.NewInt(0) + watcher.latestHeader = nil + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(err) + go tw.Watch() + time.Sleep(2 * time.Millisecond) + // Check state initialization + bh = tw.LastInitializedBlockHash() + assert.Equal(hash, common.BytesToHash(bh[:])) + assert.Equal(round, tw.LastInitializedRound()) + assert.Equal(size, tw.GetTranscoderPoolSize()) + assert.Equal(big.NewInt(0), tw.LastSeenBlock()) // New Round event watcher.sink <- []*blockwatch.Event{blockEvent} @@ -71,6 +100,7 @@ func TestRoundsWatcher_WatchAndStop(t *testing.T) { copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) assert.Equal(size, tw.GetTranscoderPoolSize()) + assert.Equal(header.Number, tw.LastSeenBlock()) // Test no NewRound events, values on rw remain the same tw.setTranscoderPoolSize(big.NewInt(10)) @@ -83,17 +113,18 @@ func TestRoundsWatcher_WatchAndStop(t *testing.T) { copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) assert.Equal(big.NewInt(10), tw.GetTranscoderPoolSize()) + assert.Equal(header.Number, tw.LastSeenBlock()) // Test RPC paths (event removed) blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) blockEvent.Type = blockwatch.Removed watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) - lastRound = tw.LastInitializedRound() bhForRound = tw.LastInitializedBlockHash() - assert.Equal(lastRound.Int64(), int64(0)) - assert.Equal(bhForRound, [32]byte{}) + assert.Equal(hash, common.BytesToHash(bh[:])) + assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) + assert.Equal(header.Number, tw.LastSeenBlock()) // Test Stop tw.Stop() @@ -103,15 +134,15 @@ func TestRoundsWatcher_WatchAndStop(t *testing.T) { // Test watch error when RPC calls fail tw = &TimeWatcher{ lpEth: ð.StubClient{ - RoundsErr: errors.New("roundswatcher error"), + RoundsErr: errors.New("timewatcher error"), }, } err = tw.Watch() assert.NotNil(err) - assert.Contains(err.Error(), "roundswatcher error") + assert.Contains(err.Error(), "timewatcher error") } -func TestRoundsWatcher_HandleLog(t *testing.T) { +func TestTimeWatcher_HandleLog(t *testing.T) { lpEth := ð.StubClient{} watcher := &stubBlockWatcher{} tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) @@ -141,7 +172,9 @@ func TestLastSeenBlock(t *testing.T) { func TestHandleBlockNum(t *testing.T) { assert := assert.New(t) - watcher := &stubBlockWatcher{} + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) @@ -162,7 +195,9 @@ func TestHandleBlockNum(t *testing.T) { func TestSubscribeBlocks(t *testing.T) { assert := assert.New(t) - watcher := &stubBlockWatcher{} + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) @@ -188,7 +223,9 @@ func TestSubscribeBlocks(t *testing.T) { func TestSubscribeRounds(t *testing.T) { assert := assert.New(t) - watcher := &stubBlockWatcher{} + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) From fac87079c83c5d14c5d4ad9e7e6b804d2dece2bf Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 10 Dec 2020 22:26:37 +0100 Subject: [PATCH 55/72] eth/watchers: add CurrentRoundStartBlock to TimeWatcher Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 1b5f53fe907088672522c72a5a1d76634088bc30 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 28 ++++++++++++++++++++++------ eth/watchers/timewatcher_test.go | 4 +++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 10f4a38..138961e 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -29,6 +29,7 @@ type TimeWatcher struct { mu sync.RWMutex lastInitializedRound *big.Int lastInitializedBlockHash [32]byte + currentRoundStartBlock *big.Int transcoderPoolSize *big.Int lastSeenBlock *big.Int @@ -75,11 +76,18 @@ func (tw *TimeWatcher) LastInitializedBlockHash() [32]byte { return tw.lastInitializedBlockHash } -func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte) { +func (tw *TimeWatcher) CurrentRoundStartBlock() *big.Int { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.currentRoundStartBlock +} + +func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, startBlk *big.Int) { tw.mu.Lock() defer tw.mu.Unlock() tw.lastInitializedRound = round tw.lastInitializedBlockHash = hash + tw.currentRoundStartBlock = startBlk } func (tw *TimeWatcher) GetTranscoderPoolSize() *big.Int { @@ -116,7 +124,11 @@ func (tw *TimeWatcher) Watch() error { if err != nil { return fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) } - tw.setLastInitializedRound(lr, bh) + num, err := tw.lpEth.CurrentRoundStartBlock() + if err != nil { + return fmt.Errorf("error fetching current round start block %v=", err) + } + tw.setLastInitializedRound(lr, bh, num) if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { return fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) @@ -209,8 +221,10 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return fmt.Errorf("unable to decode event: %v", err) } - tw.roundSubFeed.Send(log) - + roundStartBlock, err := tw.lpEth.CurrentRoundStartBlock() + if err != nil { + return err + } if log.Removed { lr, err := tw.lpEth.LastInitializedRound() if err != nil { @@ -220,9 +234,9 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { if err != nil { return err } - tw.setLastInitializedRound(lr, bh) + tw.setLastInitializedRound(lr, bh, roundStartBlock) } else { - tw.setLastInitializedRound(nr.Round, nr.BlockHash) + tw.setLastInitializedRound(nr.Round, nr.BlockHash, roundStartBlock) } // Get the active transcoder pool size when we receive a NewRound event @@ -230,6 +244,8 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return err } + tw.roundSubFeed.Send(log) + return nil } diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index ea4a2cc..f5c5f8d 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -21,7 +21,8 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { round := big.NewInt(5) var hash [32]byte copy(hash[:], "hello world") - tw.setLastInitializedRound(round, hash) + num := big.NewInt(10) + tw.setLastInitializedRound(round, hash, num) assert.Equal(tw.lastInitializedRound, round) assert.Equal(tw.lastInitializedBlockHash, hash) @@ -29,6 +30,7 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { assert.Equal(r, round) h := tw.LastInitializedBlockHash() assert.Equal(h, hash) + assert.Equal(tw.CurrentRoundStartBlock(), num) } func TestSetAndGet_TranscoderPoolSize(t *testing.T) { From 4016bbed68fbe96ab82b8023fb0a2b2585d28cac Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Thu, 7 Jan 2021 00:00:18 +0100 Subject: [PATCH 56/72] eth: blockwatcher can process multiple blocks in a polling interval Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 003029a6cfba61cef7b4222941287fecdc8084e3 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 955b6b5..4f7d241 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -68,7 +68,7 @@ type Watcher struct { ticker *time.Ticker withLogs bool topics []common.Hash - mu sync.RWMutex + sync.RWMutex } // New creates a new Watcher instance. @@ -105,13 +105,13 @@ func (w *Watcher) BackfillEventsIfNeeded(ctx context.Context) error { // Watch starts the Watcher. It will continuously look for new blocks and blocks // until the given context is canceled. Typically, you want to call Watch inside a goroutine. func (w *Watcher) Watch(ctx context.Context) error { - w.mu.Lock() + w.Lock() if w.wasStartedOnce { - w.mu.Unlock() + w.Unlock() return errors.New("Can only start Watcher once per instance") } w.wasStartedOnce = true - w.mu.Unlock() + w.Unlock() ticker := time.NewTicker(w.pollingInterval) for { @@ -120,7 +120,7 @@ func (w *Watcher) Watch(ctx context.Context) error { ticker.Stop() return nil case <-ticker.C: - if err := w.pollNextBlock(); err != nil { + if err := w.syncToLatestBlock(); err != nil { glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%v", err) } } @@ -146,6 +146,27 @@ func (w *Watcher) InspectRetainedBlocks() ([]*MiniHeader, error) { return w.stack.Inspect() } +func (w *Watcher) syncToLatestBlock() error { + w.Lock() + defer w.Unlock() + newestHeader, err := w.client.HeaderByNumber(nil) + if err != nil { + return err + } + + lastSeenHeader, err := w.stack.Peek() + if err != nil { + return err + } + + for i := lastSeenHeader.Number; i.Cmp(newestHeader.Number) < 0; i = i.Add(i, big.NewInt(1)) { + if err := w.pollNextBlock(); err != nil { + return err + } + } + return nil +} + // pollNextBlock polls for the next block header to be added to the block stack. // If there are no blocks on the stack, it fetches the first block at the specified // `startBlockDepth` supplied at instantiation. From cac59c82dce4cc9544bd2625f71d413f3a82f7b5 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Tue, 6 Apr 2021 17:32:33 +0200 Subject: [PATCH 57/72] blockwatch: don't log backfilling statement when no blocks have elapsed Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 55b14ac81132831fb85dabfe4493cc72a82aa7c6 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 4f7d241..87b7d56 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -159,6 +159,10 @@ func (w *Watcher) syncToLatestBlock() error { return err } + if lastSeenHeader == nil { + return w.pollNextBlock() + } + for i := lastSeenHeader.Number; i.Cmp(newestHeader.Number) < 0; i = i.Add(i, big.NewInt(1)) { if err := w.pollNextBlock(); err != nil { return err @@ -332,7 +336,7 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro return events, nil } - if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed == 0 { + if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed <= 0 { return events, nil } From 6c60ee6ac9c06faff4b509d7294e6ce286503562 Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 12 Apr 2021 17:05:40 +0200 Subject: [PATCH 58/72] eth: move timewatcher state initialisation into constructor to be synchronous Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 3aec3aa1628af8307315bd80123632714f9a4044 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 67 ++++++++++--------- eth/watchers/timewatcher_test.go | 110 +++++++++++++++++++++++-------- 2 files changed, 118 insertions(+), 59 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 138961e..c9fc6ca 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -54,12 +54,42 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l return nil, fmt.Errorf("error creating decoder: %v", err) } - return &TimeWatcher{ + tw := &TimeWatcher{ quit: make(chan struct{}), watcher: watcher, lpEth: lpEth, dec: dec, - }, nil + } + + lr, err := tw.lpEth.LastInitializedRound() + if err != nil { + return nil, fmt.Errorf("error fetching initial lastInitializedRound value err=%v", err) + } + bh, err := tw.lpEth.BlockHashForRound(lr) + if err != nil { + return nil, fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) + } + num, err := tw.lpEth.CurrentRoundStartBlock() + if err != nil { + return nil, fmt.Errorf("error fetching current round start block %v=", err) + } + tw.setLastInitializedRound(lr, bh, num) + + lastSeenBlock, err := tw.watcher.GetLatestBlock() + if err != nil { + return nil, fmt.Errorf("error fetching last seen block err=%v", err) + } + blockNum := big.NewInt(0) + if lastSeenBlock != nil { + blockNum = lastSeenBlock.Number + } + tw.setLastSeenBlock(blockNum) + + if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { + return nil, fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) + } + + return tw, nil } // LastInitializedRound gets the last initialized round from cache @@ -93,6 +123,9 @@ func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, st func (tw *TimeWatcher) GetTranscoderPoolSize() *big.Int { tw.mu.RLock() defer tw.mu.RUnlock() + if tw.transcoderPoolSize == nil { + return big.NewInt(0) + } return tw.transcoderPoolSize } @@ -116,34 +149,6 @@ func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { // Watch the blockwatch subscription for NewRound events func (tw *TimeWatcher) Watch() error { - lr, err := tw.lpEth.LastInitializedRound() - if err != nil { - return fmt.Errorf("error fetching initial lastInitializedRound value err=%v", err) - } - bh, err := tw.lpEth.BlockHashForRound(lr) - if err != nil { - return fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) - } - num, err := tw.lpEth.CurrentRoundStartBlock() - if err != nil { - return fmt.Errorf("error fetching current round start block %v=", err) - } - tw.setLastInitializedRound(lr, bh, num) - - if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) - } - - lastSeenBlock, err := tw.watcher.GetLatestBlock() - if err != nil { - return fmt.Errorf("error fetching last seen block err=%v", err) - } - blockNum := big.NewInt(0) - if lastSeenBlock != nil { - blockNum = lastSeenBlock.Number - } - tw.setLastSeenBlock(blockNum) - events := make(chan []*blockwatch.Event, 10) sub := tw.watcher.Subscribe(events) defer sub.Unsubscribe() @@ -252,7 +257,7 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { func (tw *TimeWatcher) fetchAndSetTranscoderPoolSize() error { size, err := tw.lpEth.GetTranscoderPoolSize() if err != nil { - return fmt.Errorf("error fetching initial transcoderPoolSize: %v", err) + return err } tw.setTranscoderPoolSize(size) return nil diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index f5c5f8d..72da00d 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -1,7 +1,7 @@ package watchers import ( - "errors" + "fmt" "math/big" "testing" "time" @@ -40,9 +40,14 @@ func TestSetAndGet_TranscoderPoolSize(t *testing.T) { tw.setTranscoderPoolSize(size) assert.Equal(size, tw.transcoderPoolSize) assert.Equal(size, tw.GetTranscoderPoolSize()) + + // return big.Int(0) when nil + tw.setTranscoderPoolSize(nil) + assert.Nil(tw.transcoderPoolSize) + assert.Equal(big.NewInt(0), tw.GetTranscoderPoolSize()) } -func TestTimeWatcher_WatchAndStop(t *testing.T) { +func TestTimeWatcher_NewTimeWatcher(t *testing.T) { assert := assert.New(t) size := big.NewInt(50) block := big.NewInt(10) @@ -53,44 +58,103 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { BlockNum: block, BlockHashToReturn: hash, Round: round, + Errors: make(map[string]error), } watcher := &stubBlockWatcher{ latestHeader: &blockwatch.MiniHeader{Number: block}, } + + testErr := fmt.Errorf("error") + + // Last InitializedRound error + lpEth.Errors["LastInitializedRound"] = testErr + expErr := fmt.Sprintf("error fetching initial lastInitializedRound value err=%v", testErr) tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) - assert.Nil(err) + assert.Nil(tw) + assert.EqualError(err, expErr) + lpEth.Errors["LastInitializedRound"] = nil - header := defaultMiniHeader() - newRoundEvent := newStubNewRoundLog() + // BlockHashForRound error + lpEth.Errors["BlockHashForRound"] = testErr + expErr = fmt.Sprintf("error fetching initial lastInitializedBlockHash value err=%v", testErr) + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(tw) + assert.EqualError(err, expErr) + lpEth.Errors["BlockHashForRound"] = nil - header.Logs = append(header.Logs, newRoundEvent) - blockEvent := &blockwatch.Event{ - Type: blockwatch.Added, - BlockHeader: header, - } + // CurrentRoundStartBlock error + lpEth.Errors["CurrentRoundStartBlock"] = testErr + expErr = fmt.Sprintf("error fetching current round start block %v=", testErr) + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(tw) + assert.EqualError(err, expErr) + lpEth.Errors["CurrentRoundStartBlock"] = nil - go tw.Watch() - time.Sleep(2 * time.Millisecond) - // Check state initialization + // GetLastestBlock error + watcher.err = fmt.Errorf("GetLatestBlock error") + expErr = fmt.Sprintf("error fetching last seen block err=%v", watcher.err) + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(tw) + assert.EqualError(err, expErr) + watcher.err = nil + + // TranscoderPoolSize error + lpEth.Errors["GetTranscoderPoolSize"] = testErr + expErr = fmt.Sprintf("error fetching initial transcoderPoolSize err=%v", testErr) + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(tw) + assert.EqualError(err, expErr) + lpEth.Errors["GetTranscoderPoolSize"] = nil + + tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + assert.Nil(err) bh := tw.LastInitializedBlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(block, tw.LastSeenBlock()) - tw.Stop() // if watcher.GetLatestBlock() == nil, initialise lastSeenBlock to big.NewInt(0) watcher.latestHeader = nil tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) - go tw.Watch() - time.Sleep(2 * time.Millisecond) - // Check state initialization bh = tw.LastInitializedBlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(big.NewInt(0), tw.LastSeenBlock()) +} + +func TestTimeWatcher_WatchAndStop(t *testing.T) { + assert := assert.New(t) + size := big.NewInt(50) + block := big.NewInt(10) + round := big.NewInt(1) + hash := ethcommon.HexToHash("foo") + lpEth := ð.StubClient{ + PoolSize: size, + BlockNum: block, + BlockHashToReturn: hash, + Round: round, + Errors: make(map[string]error), + } + watcher := &stubBlockWatcher{ + latestHeader: &blockwatch.MiniHeader{Number: block}, + } + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + require.Nil(t, err) + + header := defaultMiniHeader() + newRoundEvent := newStubNewRoundLog() + + header.Logs = append(header.Logs, newRoundEvent) + blockEvent := &blockwatch.Event{ + Type: blockwatch.Added, + BlockHeader: header, + } + + go tw.Watch() + time.Sleep(2 * time.Millisecond) // New Round event watcher.sink <- []*blockwatch.Event{blockEvent} @@ -123,7 +187,7 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) bhForRound = tw.LastInitializedBlockHash() - assert.Equal(hash, common.BytesToHash(bh[:])) + assert.Equal(hash, common.BytesToHash(bhForRound[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(header.Number, tw.LastSeenBlock()) @@ -132,16 +196,6 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { tw.Stop() time.Sleep(2 * time.Millisecond) assert.True(watcher.sub.unsubscribed) - - // Test watch error when RPC calls fail - tw = &TimeWatcher{ - lpEth: ð.StubClient{ - RoundsErr: errors.New("timewatcher error"), - }, - } - err = tw.Watch() - assert.NotNil(err) - assert.Contains(err.Error(), "timewatcher error") } func TestTimeWatcher_HandleLog(t *testing.T) { From 61b7d16445134618f66501a6ebfdc9cd88c3133e Mon Sep 17 00:00:00 2001 From: Nico Vergauwen Date: Mon, 12 Apr 2021 17:12:54 +0200 Subject: [PATCH 59/72] eth: timewatcher code cleanup, remove fetchAndSetTranscoderPoolSize Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 3719aeed2f9dbf05cbba0c0d2873861f160cca73 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 19 +++++++------------ eth/watchers/timewatcher_test.go | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index c9fc6ca..11c09d2 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -71,7 +71,7 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l } num, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { - return nil, fmt.Errorf("error fetching current round start block %v=", err) + return nil, fmt.Errorf("error fetching current round start block err=%v", err) } tw.setLastInitializedRound(lr, bh, num) @@ -85,9 +85,11 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l } tw.setLastSeenBlock(blockNum) - if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { + size, err := tw.lpEth.GetTranscoderPoolSize() + if err != nil { return nil, fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) } + tw.setTranscoderPoolSize(size) return tw, nil } @@ -245,20 +247,13 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { } // Get the active transcoder pool size when we receive a NewRound event - if err := tw.fetchAndSetTranscoderPoolSize(); err != nil { - return err - } - - tw.roundSubFeed.Send(log) - - return nil -} - -func (tw *TimeWatcher) fetchAndSetTranscoderPoolSize() error { size, err := tw.lpEth.GetTranscoderPoolSize() if err != nil { return err } tw.setTranscoderPoolSize(size) + + tw.roundSubFeed.Send(log) + return nil } diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index 72da00d..05a31c2 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -84,7 +84,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // CurrentRoundStartBlock error lpEth.Errors["CurrentRoundStartBlock"] = testErr - expErr = fmt.Sprintf("error fetching current round start block %v=", testErr) + expErr = fmt.Sprintf("error fetching current round start block err=%v", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) From 856697038b17d7248721f6a08305a258aa9a48cb Mon Sep 17 00:00:00 2001 From: Ivan Tivonenko Date: Tue, 30 Nov 2021 19:45:56 +0200 Subject: [PATCH 60/72] quote errors Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: c2c73fa6f24473311766baee83e2762c87ef708d Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 2 +- eth/watchers/timewatcher.go | 10 +++++----- eth/watchers/timewatcher_test.go | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 87b7d56..6e23aad 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -121,7 +121,7 @@ func (w *Watcher) Watch(ctx context.Context) error { return nil case <-ticker.C: if err := w.syncToLatestBlock(); err != nil { - glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%v", err) + glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%q", err) } } } diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 11c09d2..7827e33 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -63,21 +63,21 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l lr, err := tw.lpEth.LastInitializedRound() if err != nil { - return nil, fmt.Errorf("error fetching initial lastInitializedRound value err=%v", err) + return nil, fmt.Errorf("error fetching initial lastInitializedRound value err=%q", err) } bh, err := tw.lpEth.BlockHashForRound(lr) if err != nil { - return nil, fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%v", err) + return nil, fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%q", err) } num, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { - return nil, fmt.Errorf("error fetching current round start block err=%v", err) + return nil, fmt.Errorf("error fetching current round start block err=%q", err) } tw.setLastInitializedRound(lr, bh, num) lastSeenBlock, err := tw.watcher.GetLatestBlock() if err != nil { - return nil, fmt.Errorf("error fetching last seen block err=%v", err) + return nil, fmt.Errorf("error fetching last seen block err=%q", err) } blockNum := big.NewInt(0) if lastSeenBlock != nil { @@ -87,7 +87,7 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l size, err := tw.lpEth.GetTranscoderPoolSize() if err != nil { - return nil, fmt.Errorf("error fetching initial transcoderPoolSize err=%v", err) + return nil, fmt.Errorf("error fetching initial transcoderPoolSize err=%q", err) } tw.setTranscoderPoolSize(size) diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index 05a31c2..a618719 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -68,7 +68,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // Last InitializedRound error lpEth.Errors["LastInitializedRound"] = testErr - expErr := fmt.Sprintf("error fetching initial lastInitializedRound value err=%v", testErr) + expErr := fmt.Sprintf("error fetching initial lastInitializedRound value err=%q", testErr) tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) @@ -76,7 +76,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // BlockHashForRound error lpEth.Errors["BlockHashForRound"] = testErr - expErr = fmt.Sprintf("error fetching initial lastInitializedBlockHash value err=%v", testErr) + expErr = fmt.Sprintf("error fetching initial lastInitializedBlockHash value err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) @@ -84,7 +84,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // CurrentRoundStartBlock error lpEth.Errors["CurrentRoundStartBlock"] = testErr - expErr = fmt.Sprintf("error fetching current round start block err=%v", testErr) + expErr = fmt.Sprintf("error fetching current round start block err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) @@ -92,7 +92,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // GetLastestBlock error watcher.err = fmt.Errorf("GetLatestBlock error") - expErr = fmt.Sprintf("error fetching last seen block err=%v", watcher.err) + expErr = fmt.Sprintf("error fetching last seen block err=%q", watcher.err) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) @@ -100,7 +100,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // TranscoderPoolSize error lpEth.Errors["GetTranscoderPoolSize"] = testErr - expErr = fmt.Sprintf("error fetching initial transcoderPoolSize err=%v", testErr) + expErr = fmt.Sprintf("error fetching initial transcoderPoolSize err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) From 9d013d549680af47017baa201364094052eb585b Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Thu, 20 Jan 2022 15:55:03 +0100 Subject: [PATCH 61/72] eth: Handle watcher feeds asynchronously Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 87c6e404c7a8bd5552e66e346d3659d696e17f4f Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 7827e33..a6ed4f4 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -161,7 +161,9 @@ func (tw *TimeWatcher) Watch() error { case err := <-sub.Err(): glog.Error(err) case events := <-events: - tw.handleBlockEvents(events) + go func() { + tw.handleBlockEvents(events) + }() } } } From 15515d5f735cae955bca5d534e6e7c6f367ec943 Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Fri, 21 Jan 2022 09:11:42 +0100 Subject: [PATCH 62/72] eth: Refactor sink naming in the watcher feeds Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: f30819e43a6b0d9272301a402220d7df64d70063 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index a6ed4f4..5207755 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -151,8 +151,8 @@ func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { // Watch the blockwatch subscription for NewRound events func (tw *TimeWatcher) Watch() error { - events := make(chan []*blockwatch.Event, 10) - sub := tw.watcher.Subscribe(events) + blockSink := make(chan []*blockwatch.Event, 10) + sub := tw.watcher.Subscribe(blockSink) defer sub.Unsubscribe() for { select { @@ -160,9 +160,9 @@ func (tw *TimeWatcher) Watch() error { return nil case err := <-sub.Err(): glog.Error(err) - case events := <-events: + case block := <-blockSink: go func() { - tw.handleBlockEvents(events) + tw.handleBlockEvents(block) }() } } From c7b4207310f7c7c14879199e2d9d7c64e0fea4b8 Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Tue, 25 Jan 2022 10:39:51 +0100 Subject: [PATCH 63/72] eth: Add locks to all feed receiver operations Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 39585360f2cd0c0626ad483e9dd677efb6d09387 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 5207755..496c508 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -39,6 +39,7 @@ type TimeWatcher struct { // last seen block number subscription feeds blockSubFeed event.Feed blockSubScope event.SubscriptionScope + feedMu sync.Mutex watcher BlockWatcher lpEth eth.LivepeerEthClient @@ -206,6 +207,9 @@ func (tw *TimeWatcher) handleBlockEvents(events []*blockwatch.Event) { } func (tw *TimeWatcher) handleBlockNum(event *blockwatch.Event) { + tw.feedMu.Lock() + defer tw.feedMu.Unlock() + last := tw.LastSeenBlock() new := event.BlockHeader.Number if last == nil || last.Cmp(new) != 0 { @@ -221,6 +225,9 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return nil } + tw.feedMu.Lock() + defer tw.feedMu.Unlock() + if eventName != "NewRound" { return fmt.Errorf("eventName is not NewRound") } From fc5f9bbd4b3e2ff5560679bb22f8b128aba08631 Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Thu, 27 Jan 2022 15:48:47 +0100 Subject: [PATCH 64/72] eth: Refactor go func() {} into go Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 754c1ed203539275332363eb2cd2940a7fa6c72a Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 496c508..f79e686 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -162,9 +162,7 @@ func (tw *TimeWatcher) Watch() error { case err := <-sub.Err(): glog.Error(err) case block := <-blockSink: - go func() { - tw.handleBlockEvents(block) - }() + go tw.handleBlockEvents(block) } } } From 26113ba314b802c7e18e96f957e9fb61fb7af72a Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Mon, 31 Jan 2022 16:55:09 +0100 Subject: [PATCH 65/72] eth,pm,server: Use L1 block number for Ticket Parameters and Round Initialization Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 4b6ede31f040a084ff9e55bd798a0d6fffee1e1b Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 42 ++++++++++++++-- eth/blockwatch/block_watcher_test.go | 73 +++++++++++++++++++++++++++ eth/watchers/timewatcher.go | 74 ++++++++++++++-------------- eth/watchers/timewatcher_test.go | 60 +++++++++++----------- 4 files changed, 179 insertions(+), 70 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 6e23aad..b89fc8d 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -97,7 +97,7 @@ func (w *Watcher) BackfillEventsIfNeeded(ctx context.Context) error { return err } if len(events) > 0 { - w.blockFeed.Send(events) + w.blockFeed.Send(w.enrichWithL1(events)) } return nil } @@ -137,7 +137,11 @@ func (w *Watcher) Subscribe(sink chan<- []*Event) event.Subscription { // GetLatestBlock returns the latest block processed func (w *Watcher) GetLatestBlock() (*MiniHeader, error) { - return w.stack.Peek() + h, err := w.stack.Peek() + if err != nil { + return nil, err + } + return w.enrichWithL1BlockNumber(h) } // InspectRetainedBlocks returns the blocks retained in-memory by the Watcher instance. It is not @@ -202,7 +206,7 @@ func (w *Watcher) pollNextBlock() error { // Even if an error occurred, we still want to emit the events gathered since we might have // popped blocks off the Stack and they won't be re-added if len(events) != 0 { - w.blockFeed.Send(events) + w.blockFeed.Send(w.enrichWithL1(events)) } if err != nil { return err @@ -609,6 +613,38 @@ func (w *Watcher) filterLogsRecursively(from, to int, allLogs []types.Log) ([]ty return allLogs, nil } +// enrichWithL1 adds L1 block number to the event with the highest block number. +// +// Adding L1 block number to an event requires an RPC call and the only code using L1 block number is TimeWatcher that +// is interested only in the highest L1 block number, not in all of them. +func (w *Watcher) enrichWithL1(events []*Event) []*Event { + if len(events) == 0 { + return events + } + + max := 0 + for i, e := range events { + if events[max].BlockHeader.Number.Cmp(e.BlockHeader.Number) <= 0 { + max = i + } + } + + block, err := w.enrichWithL1BlockNumber(events[max].BlockHeader) + if err != nil { + glog.Errorf("Cannot fetch L1 block number for block number %d err=%q", events[max].BlockHeader.Number, err) + } + + events[max].BlockHeader = block + return events +} + +func (w *Watcher) enrichWithL1BlockNumber(header *MiniHeader) (*MiniHeader, error) { + if header == nil || header.L1BlockNumber != nil { + return header, nil + } + return w.client.HeaderByHash(header.Hash) +} + func isUnknownBlockErr(err error) bool { // Geth error if err.Error() == "unknown block" { diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 06e4311..3e4b88e 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -551,3 +551,76 @@ func aRange(from, to int) string { r := fmt.Sprintf("%d-%d", from, to) return r } + +func TestEnrichWithL1_Empty(t *testing.T) { + assert := assert.New(t) + + w := New(config) + res := w.enrichWithL1([]*Event{}) + assert.NotNil(res) + assert.Empty(res) +} + +func TestEnrichWithL1_One(t *testing.T) { + assert := assert.New(t) + + config.Client = &fakeClient{} + w := New(config) + header := &MiniHeader{ + Number: big.NewInt(FakeBlockNumber), + Hash: common.HexToHash(FakeHash), + } + events := []*Event{{BlockHeader: header}} + + res := w.enrichWithL1(events) + + assert.Len(res, 1) + assert.Equal(common.HexToHash(FakeHash), res[0].BlockHeader.Hash) + assert.Equal(big.NewInt(FakeBlockNumber), res[0].BlockHeader.Number) + assert.Equal(big.NewInt(FakeL1BlockNumber), res[0].BlockHeader.L1BlockNumber) +} + +func TestEnrichWithL1_Multiple(t *testing.T) { + assert := assert.New(t) + + config.Client = &fakeClient{} + w := New(config) + highestHeader := &MiniHeader{ + Number: big.NewInt(FakeBlockNumber), + Hash: common.HexToHash(FakeHash), + } + events := []*Event{ + {BlockHeader: &MiniHeader{Number: big.NewInt(1)}}, + {BlockHeader: highestHeader}, + {BlockHeader: &MiniHeader{Number: big.NewInt(2)}}, + } + + res := w.enrichWithL1(events) + + assert.Len(res, 3) + // highest block number event has L1 block number + assert.Equal(common.HexToHash(FakeHash), res[1].BlockHeader.Hash) + assert.Equal(big.NewInt(FakeBlockNumber), res[1].BlockHeader.Number) + assert.Equal(big.NewInt(FakeL1BlockNumber), res[1].BlockHeader.L1BlockNumber) + // other events do not have L1 block numbers + assert.Nil(res[0].BlockHeader.L1BlockNumber) + assert.Nil(res[2].BlockHeader.L1BlockNumber) +} + +func TestEnrichWithL1BlockNumber(t *testing.T) { + assert := assert.New(t) + + config.Client = &fakeClient{} + w := New(config) + header := &MiniHeader{ + Number: big.NewInt(FakeBlockNumber), + Hash: common.HexToHash(FakeHash), + } + + res, err := w.enrichWithL1BlockNumber(header) + + assert.NoError(err) + assert.Equal(common.HexToHash(FakeHash), res.Hash) + assert.Equal(big.NewInt(FakeBlockNumber), res.Number) + assert.Equal(big.NewInt(FakeL1BlockNumber), res.L1BlockNumber) +} diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index f79e686..28f8322 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -26,20 +26,20 @@ import ( // * Last Seen Block Number type TimeWatcher struct { // state - mu sync.RWMutex - lastInitializedRound *big.Int - lastInitializedBlockHash [32]byte - currentRoundStartBlock *big.Int - transcoderPoolSize *big.Int - lastSeenBlock *big.Int + mu sync.RWMutex + lastInitializedRound *big.Int + lastInitializedL1BlockHash [32]byte + currentRoundStartL1Block *big.Int + transcoderPoolSize *big.Int + lastSeenL1Block *big.Int // last initialized round number subscription feeds roundSubFeed event.Feed roundSubScope event.SubscriptionScope // last seen block number subscription feeds - blockSubFeed event.Feed - blockSubScope event.SubscriptionScope - feedMu sync.Mutex + l1BlockSubFeed event.Feed + l1BlockSubScope event.SubscriptionScope + feedMu sync.Mutex watcher BlockWatcher lpEth eth.LivepeerEthClient @@ -68,7 +68,7 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l } bh, err := tw.lpEth.BlockHashForRound(lr) if err != nil { - return nil, fmt.Errorf("error fetching initial lastInitializedBlockHash value err=%q", err) + return nil, fmt.Errorf("error fetching initial lastInitializedL1BlockHash value err=%q", err) } num, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { @@ -80,11 +80,11 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l if err != nil { return nil, fmt.Errorf("error fetching last seen block err=%q", err) } - blockNum := big.NewInt(0) + l1BlockNum := big.NewInt(0) if lastSeenBlock != nil { - blockNum = lastSeenBlock.Number + l1BlockNum = lastSeenBlock.L1BlockNumber } - tw.setLastSeenBlock(blockNum) + tw.setLastSeenL1Block(l1BlockNum) size, err := tw.lpEth.GetTranscoderPoolSize() if err != nil { @@ -103,24 +103,24 @@ func (tw *TimeWatcher) LastInitializedRound() *big.Int { } // LastInitializedBlockHash returns the blockhash of the block the last round was initiated in -func (tw *TimeWatcher) LastInitializedBlockHash() [32]byte { +func (tw *TimeWatcher) LastInitializedL1BlockHash() [32]byte { tw.mu.RLock() defer tw.mu.RUnlock() - return tw.lastInitializedBlockHash + return tw.lastInitializedL1BlockHash } -func (tw *TimeWatcher) CurrentRoundStartBlock() *big.Int { +func (tw *TimeWatcher) CurrentRoundStartL1Block() *big.Int { tw.mu.RLock() defer tw.mu.RUnlock() - return tw.currentRoundStartBlock + return tw.currentRoundStartL1Block } func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, startBlk *big.Int) { tw.mu.Lock() defer tw.mu.Unlock() tw.lastInitializedRound = round - tw.lastInitializedBlockHash = hash - tw.currentRoundStartBlock = startBlk + tw.lastInitializedL1BlockHash = hash + tw.currentRoundStartL1Block = startBlk } func (tw *TimeWatcher) GetTranscoderPoolSize() *big.Int { @@ -138,16 +138,16 @@ func (tw *TimeWatcher) setTranscoderPoolSize(size *big.Int) { tw.transcoderPoolSize = size } -func (tw *TimeWatcher) LastSeenBlock() *big.Int { +func (tw *TimeWatcher) LastSeenL1Block() *big.Int { tw.mu.RLock() defer tw.mu.RUnlock() - return tw.lastSeenBlock + return tw.lastSeenL1Block } -func (tw *TimeWatcher) setLastSeenBlock(blockNum *big.Int) { +func (tw *TimeWatcher) setLastSeenL1Block(blockNum *big.Int) { tw.mu.Lock() defer tw.mu.Unlock() - tw.lastSeenBlock = blockNum + tw.lastSeenL1Block = blockNum } // Watch the blockwatch subscription for NewRound events @@ -175,24 +175,24 @@ func (tw *TimeWatcher) SubscribeRounds(sink chan<- types.Log) event.Subscription return tw.roundSubScope.Track(tw.roundSubFeed.Subscribe(sink)) } -// SubscribeBlocks allows one to subscribe to newly seen block numbers +// SubscribeL1Blocks allows one to subscribe to newly seen L1 block numbers // To unsubscribe, simply call `Unsubscribe` on the returned subscription. // The sink channel should have ample buffer space to avoid blocking other subscribers. // Slow subscribers are not dropped. -func (tw *TimeWatcher) SubscribeBlocks(sink chan<- *big.Int) event.Subscription { - return tw.blockSubScope.Track(tw.blockSubFeed.Subscribe(sink)) +func (tw *TimeWatcher) SubscribeL1Blocks(sink chan<- *big.Int) event.Subscription { + return tw.l1BlockSubScope.Track(tw.l1BlockSubFeed.Subscribe(sink)) } // Stop TimeWatcher func (tw *TimeWatcher) Stop() { close(tw.quit) - tw.blockSubScope.Close() + tw.l1BlockSubScope.Close() tw.roundSubScope.Close() } func (tw *TimeWatcher) handleBlockEvents(events []*blockwatch.Event) { for _, event := range events { - tw.handleBlockNum(event) + tw.handleL1BlockNum(event) for _, log := range event.BlockHeader.Logs { if event.Type == blockwatch.Removed { log.Removed = true @@ -204,15 +204,15 @@ func (tw *TimeWatcher) handleBlockEvents(events []*blockwatch.Event) { } } -func (tw *TimeWatcher) handleBlockNum(event *blockwatch.Event) { +func (tw *TimeWatcher) handleL1BlockNum(event *blockwatch.Event) { tw.feedMu.Lock() defer tw.feedMu.Unlock() - last := tw.LastSeenBlock() - new := event.BlockHeader.Number - if last == nil || last.Cmp(new) != 0 { - tw.setLastSeenBlock(new) - tw.blockSubFeed.Send(new) + last := tw.LastSeenL1Block() + new := event.BlockHeader.L1BlockNumber + if new != nil && (last == nil || last.Cmp(new) != 0) { + tw.setLastSeenL1Block(new) + tw.l1BlockSubFeed.Send(new) } } @@ -235,7 +235,7 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return fmt.Errorf("unable to decode event: %v", err) } - roundStartBlock, err := tw.lpEth.CurrentRoundStartBlock() + roundStartL1Block, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { return err } @@ -248,9 +248,9 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { if err != nil { return err } - tw.setLastInitializedRound(lr, bh, roundStartBlock) + tw.setLastInitializedRound(lr, bh, roundStartL1Block) } else { - tw.setLastInitializedRound(nr.Round, nr.BlockHash, roundStartBlock) + tw.setLastInitializedRound(nr.Round, nr.BlockHash, roundStartL1Block) } // Get the active transcoder pool size when we receive a NewRound event diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index a618719..73f6123 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -24,13 +24,13 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { num := big.NewInt(10) tw.setLastInitializedRound(round, hash, num) assert.Equal(tw.lastInitializedRound, round) - assert.Equal(tw.lastInitializedBlockHash, hash) + assert.Equal(tw.lastInitializedL1BlockHash, hash) r := tw.LastInitializedRound() assert.Equal(r, round) - h := tw.LastInitializedBlockHash() + h := tw.LastInitializedL1BlockHash() assert.Equal(h, hash) - assert.Equal(tw.CurrentRoundStartBlock(), num) + assert.Equal(tw.CurrentRoundStartL1Block(), num) } func TestSetAndGet_TranscoderPoolSize(t *testing.T) { @@ -61,7 +61,7 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { Errors: make(map[string]error), } watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: block}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: block}, } testErr := fmt.Errorf("error") @@ -76,13 +76,13 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { // BlockHashForRound error lpEth.Errors["BlockHashForRound"] = testErr - expErr = fmt.Sprintf("error fetching initial lastInitializedBlockHash value err=%q", testErr) + expErr = fmt.Sprintf("error fetching initial lastInitializedL1BlockHash value err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(tw) assert.EqualError(err, expErr) lpEth.Errors["BlockHashForRound"] = nil - // CurrentRoundStartBlock error + // CurrentRoundStartL1Block error lpEth.Errors["CurrentRoundStartBlock"] = testErr expErr = fmt.Sprintf("error fetching current round start block err=%q", testErr) tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) @@ -108,21 +108,21 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) - bh := tw.LastInitializedBlockHash() + bh := tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) - assert.Equal(block, tw.LastSeenBlock()) + assert.Equal(block, tw.LastSeenL1Block()) - // if watcher.GetLatestBlock() == nil, initialise lastSeenBlock to big.NewInt(0) + // if watcher.GetLatestBlock() == nil, initialise lastSeenL1Block to big.NewInt(0) watcher.latestHeader = nil tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) - bh = tw.LastInitializedBlockHash() + bh = tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) - assert.Equal(big.NewInt(0), tw.LastSeenBlock()) + assert.Equal(big.NewInt(0), tw.LastSeenL1Block()) } func TestTimeWatcher_WatchAndStop(t *testing.T) { @@ -139,7 +139,7 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { Errors: make(map[string]error), } watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: block}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: block}, } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) require.Nil(t, err) @@ -161,12 +161,12 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { time.Sleep(2 * time.Millisecond) lastRound := tw.LastInitializedRound() assert.Zero(lastRound.Cmp(big.NewInt(8))) - bhForRound := tw.LastInitializedBlockHash() + bhForRound := tw.LastInitializedL1BlockHash() var expectedHashForRound [32]byte copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) assert.Equal(size, tw.GetTranscoderPoolSize()) - assert.Equal(header.Number, tw.LastSeenBlock()) + assert.Equal(header.L1BlockNumber, tw.LastSeenL1Block()) // Test no NewRound events, values on rw remain the same tw.setTranscoderPoolSize(big.NewInt(10)) @@ -175,22 +175,22 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { time.Sleep(2 * time.Millisecond) lastRound = tw.LastInitializedRound() assert.Zero(lastRound.Cmp(big.NewInt(8))) - bhForRound = tw.LastInitializedBlockHash() + bhForRound = tw.LastInitializedL1BlockHash() copy(expectedHashForRound[:], newRoundEvent.Data[:]) assert.Equal(bhForRound, expectedHashForRound) assert.Equal(big.NewInt(10), tw.GetTranscoderPoolSize()) - assert.Equal(header.Number, tw.LastSeenBlock()) + assert.Equal(header.L1BlockNumber, tw.LastSeenL1Block()) // Test RPC paths (event removed) blockEvent.BlockHeader.Logs = append(blockEvent.BlockHeader.Logs, newRoundEvent) blockEvent.Type = blockwatch.Removed watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) - bhForRound = tw.LastInitializedBlockHash() + bhForRound = tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bhForRound[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) - assert.Equal(header.Number, tw.LastSeenBlock()) + assert.Equal(header.L1BlockNumber, tw.LastSeenL1Block()) // Test Stop tw.Stop() @@ -213,7 +213,7 @@ func TestTimeWatcher_HandleLog(t *testing.T) { err = tw.handleLog(log) assert.Nil(err) assert.Nil(tw.LastInitializedRound()) - assert.Equal([32]byte{}, tw.LastInitializedBlockHash()) + assert.Equal([32]byte{}, tw.LastInitializedL1BlockHash()) } func TestLastSeenBlock(t *testing.T) { @@ -221,21 +221,21 @@ func TestLastSeenBlock(t *testing.T) { tw := &TimeWatcher{} block := big.NewInt(5) - tw.setLastSeenBlock(block) - tw.LastSeenBlock() - assert.Equal(big.NewInt(5), tw.LastSeenBlock()) + tw.setLastSeenL1Block(block) + tw.LastSeenL1Block() + assert.Equal(big.NewInt(5), tw.LastSeenL1Block()) } func TestHandleBlockNum(t *testing.T) { assert := assert.New(t) watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) header := defaultMiniHeader() - header.Number = big.NewInt(10) + header.L1BlockNumber = big.NewInt(10) blockEvent := &blockwatch.Event{ Type: blockwatch.Added, BlockHeader: header, @@ -246,26 +246,26 @@ func TestHandleBlockNum(t *testing.T) { watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) - assert.Equal(tw.LastSeenBlock(), header.Number) + assert.Equal(tw.LastSeenL1Block(), header.L1BlockNumber) } func TestSubscribeBlocks(t *testing.T) { assert := assert.New(t) watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) assert.Nil(err) header := defaultMiniHeader() - header.Number = big.NewInt(10) + header.L1BlockNumber = big.NewInt(10) blockEvent := &blockwatch.Event{ Type: blockwatch.Added, BlockHeader: header, } events := make(chan *big.Int, 10) - sub := tw.SubscribeBlocks(events) + sub := tw.SubscribeL1Blocks(events) defer sub.Unsubscribe() go tw.Watch() @@ -274,13 +274,13 @@ func TestSubscribeBlocks(t *testing.T) { watcher.sink <- []*blockwatch.Event{blockEvent} time.Sleep(2 * time.Millisecond) update := <-events - assert.Equal(update, header.Number) + assert.Equal(update, header.L1BlockNumber) } func TestSubscribeRounds(t *testing.T) { assert := assert.New(t) watcher := &stubBlockWatcher{ - latestHeader: &blockwatch.MiniHeader{Number: big.NewInt(1)}, + latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) From bd8663e83fd16afb02f14d2c6eeb7fd78969f572 Mon Sep 17 00:00:00 2001 From: Rafal Leszko Date: Fri, 4 Feb 2022 13:32:14 +0100 Subject: [PATCH 66/72] eth,cmd: Backfill always from the back last seen in DB (instead of the last initialization round block) Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: a75e8e26d8e3880724c80f212a2568febd243d52 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 7 +---- eth/blockwatch/block_watcher_test.go | 42 ---------------------------- 2 files changed, 1 insertion(+), 48 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index b89fc8d..084676e 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -45,7 +45,6 @@ type Config struct { Store MiniHeaderStore PollingInterval time.Duration StartBlockDepth rpc.BlockNumber - BackfillStartBlock *big.Int BlockRetentionLimit int WithLogs bool Topics []common.Hash @@ -58,7 +57,6 @@ type Config struct { type Watcher struct { blockRetentionLimit int startBlockDepth rpc.BlockNumber - backfillStartBlock *big.Int stack *Stack client Client blockFeed event.Feed @@ -79,7 +77,6 @@ func New(config Config) *Watcher { pollingInterval: config.PollingInterval, blockRetentionLimit: config.BlockRetentionLimit, startBlockDepth: config.StartBlockDepth, - backfillStartBlock: config.BackfillStartBlock, stack: stack, client: config.Client, withLogs: config.WithLogs, @@ -330,9 +327,7 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro } latestBlockNum := int(latestBlock.Number.Int64()) - if w.backfillStartBlock != nil { - startBlockNum = int(w.backfillStartBlock.Int64()) - } else if latestRetainedBlock != nil { + if latestRetainedBlock != nil { latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) // Events for latestRetainedBlock already processed, start at latestRetainedBlock + 1 startBlockNum = latestRetainedBlockNum + 1 diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 3e4b88e..7fc05df 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -214,48 +214,6 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { assert.Equal(t, big.NewInt(30), headers[0].Number) } -func TestGetMissedEventsToBackfill_BackfillStartBlock(t *testing.T) { - // Fixture will return block 30 as the tip of the chain - fakeClient, err := newFakeClient("testdata/fake_client_fast_sync_fixture.json") - require.NoError(t, err) - - store := &stubMiniHeaderStore{} - - config.Store = store - config.Client = fakeClient - config.BackfillStartBlock = big.NewInt(0) - watcher := New(config) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - events, err := watcher.getMissedEventsToBackfill(ctx) - require.NoError(t, err) - assert.Len(t, events, 1) - - // Check that block 30 is now in the DB as it is the last processed block - headers, err := store.FindAllMiniHeadersSortedByNumber() - require.NoError(t, err) - require.Len(t, headers, 1) - assert.Equal(t, big.NewInt(30), headers[0].Number) - - store = &stubMiniHeaderStore{} - config.Store = store - config.BackfillStartBlock = big.NewInt(5) - watcher = New(config) - events, err = watcher.getMissedEventsToBackfill(ctx) - require.NoError(t, err) - assert.Len(t, events, 1) - - // Check that block 30 is now in the DB as it is the last processed block - headers, err = store.FindAllMiniHeadersSortedByNumber() - require.NoError(t, err) - require.Len(t, headers, 1) - assert.Equal(t, big.NewInt(30), headers[0].Number) - - config.BackfillStartBlock = nil -} - func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { // Fixture will return block 5 as the tip of the chain fakeClient, err := newFakeClient("testdata/fake_client_basic_fixture.json") From a52452f14d7842c7ed420c645a5799ab0dd546d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Wed, 16 Feb 2022 16:39:12 +0100 Subject: [PATCH 67/72] Fix nil pointer in the block header logs Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 17a93a67b9cf3409b47e4e01797b2416cb6d234c Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 7 ++++++- eth/blockwatch/block_watcher_test.go | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 084676e..c097b58 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -637,7 +637,12 @@ func (w *Watcher) enrichWithL1BlockNumber(header *MiniHeader) (*MiniHeader, erro if header == nil || header.L1BlockNumber != nil { return header, nil } - return w.client.HeaderByHash(header.Hash) + fetchedBlock, err := w.client.HeaderByHash(header.Hash) + if err != nil { + return header, err + } + header.L1BlockNumber = fetchedBlock.L1BlockNumber + return header, nil } func isUnknownBlockErr(err error) bool { diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 7fc05df..f94f68b 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -527,6 +527,7 @@ func TestEnrichWithL1_One(t *testing.T) { header := &MiniHeader{ Number: big.NewInt(FakeBlockNumber), Hash: common.HexToHash(FakeHash), + Logs: []types.Log{logStub}, } events := []*Event{{BlockHeader: header}} @@ -536,6 +537,7 @@ func TestEnrichWithL1_One(t *testing.T) { assert.Equal(common.HexToHash(FakeHash), res[0].BlockHeader.Hash) assert.Equal(big.NewInt(FakeBlockNumber), res[0].BlockHeader.Number) assert.Equal(big.NewInt(FakeL1BlockNumber), res[0].BlockHeader.L1BlockNumber) + assert.Contains(res[0].BlockHeader.Logs, logStub) } func TestEnrichWithL1_Multiple(t *testing.T) { From 14dde614b3878b881b1a0503391b3594cd9b1e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Wed, 13 Apr 2022 15:52:13 +0200 Subject: [PATCH 68/72] eth, pm: Backdate tickets by one round if the block hash for the current round is 0 (#2362) Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 11b0600fc76595799205a1229614a1dce954d787 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 56 ++++++++++++++++---- eth/watchers/timewatcher_test.go | 90 +++++++++++++++++++++++++++++--- 2 files changed, 129 insertions(+), 17 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 28f8322..4fbcd24 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -26,12 +26,13 @@ import ( // * Last Seen Block Number type TimeWatcher struct { // state - mu sync.RWMutex - lastInitializedRound *big.Int - lastInitializedL1BlockHash [32]byte - currentRoundStartL1Block *big.Int - transcoderPoolSize *big.Int - lastSeenL1Block *big.Int + mu sync.RWMutex + lastInitializedRound *big.Int + lastInitializedL1BlockHash [32]byte + preLastInitializedL1BlockHash [32]byte + currentRoundStartL1Block *big.Int + transcoderPoolSize *big.Int + lastSeenL1Block *big.Int // last initialized round number subscription feeds roundSubFeed event.Feed @@ -70,11 +71,15 @@ func NewTimeWatcher(roundsManagerAddr ethcommon.Address, watcher BlockWatcher, l if err != nil { return nil, fmt.Errorf("error fetching initial lastInitializedL1BlockHash value err=%q", err) } + pbh, err := tw.lpEth.BlockHashForRound(prevRound(lr)) + if err != nil { + return nil, fmt.Errorf("error fetching initial preLastInitializedL1BlockHash value err=%q", err) + } num, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { return nil, fmt.Errorf("error fetching current round start block err=%q", err) } - tw.setLastInitializedRound(lr, bh, num) + tw.setLastInitializedRound(lr, bh, pbh, num) lastSeenBlock, err := tw.watcher.GetLatestBlock() if err != nil { @@ -102,24 +107,32 @@ func (tw *TimeWatcher) LastInitializedRound() *big.Int { return tw.lastInitializedRound } -// LastInitializedBlockHash returns the blockhash of the block the last round was initiated in +// LastInitializedL1BlockHash returns the blockhash of the L1 block the last round was initiated in func (tw *TimeWatcher) LastInitializedL1BlockHash() [32]byte { tw.mu.RLock() defer tw.mu.RUnlock() return tw.lastInitializedL1BlockHash } +// PreLastInitializedL1BlockHash returns the blockhash of the L1 block for the round proceeding the last initialized round +func (tw *TimeWatcher) PreLastInitializedL1BlockHash() [32]byte { + tw.mu.RLock() + defer tw.mu.RUnlock() + return tw.preLastInitializedL1BlockHash +} + func (tw *TimeWatcher) CurrentRoundStartL1Block() *big.Int { tw.mu.RLock() defer tw.mu.RUnlock() return tw.currentRoundStartL1Block } -func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, startBlk *big.Int) { +func (tw *TimeWatcher) setLastInitializedRound(round *big.Int, hash [32]byte, preHash [32]byte, startBlk *big.Int) { tw.mu.Lock() defer tw.mu.Unlock() tw.lastInitializedRound = round tw.lastInitializedL1BlockHash = hash + tw.preLastInitializedL1BlockHash = preHash tw.currentRoundStartL1Block = startBlk } @@ -235,6 +248,10 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return fmt.Errorf("unable to decode event: %v", err) } + return tw.handleDecodedLog(log, nr) +} + +func (tw *TimeWatcher) handleDecodedLog(log types.Log, nr contracts.RoundsManagerNewRound) error { roundStartL1Block, err := tw.lpEth.CurrentRoundStartBlock() if err != nil { return err @@ -248,9 +265,22 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { if err != nil { return err } - tw.setLastInitializedRound(lr, bh, roundStartL1Block) + pbh, err := tw.lpEth.BlockHashForRound(prevRound(lr)) + if err != nil { + return err + } + tw.setLastInitializedRound(lr, bh, pbh, roundStartL1Block) } else { - tw.setLastInitializedRound(nr.Round, nr.BlockHash, roundStartL1Block) + var pbh [32]byte + if nr.Round.Cmp(big.NewInt(0).Add(tw.lastInitializedRound, big.NewInt(1))) == 0 { + pbh = tw.lastInitializedL1BlockHash + } else { + pbh, err = tw.lpEth.BlockHashForRound(prevRound(nr.Round)) + if err != nil { + return err + } + } + tw.setLastInitializedRound(nr.Round, nr.BlockHash, pbh, roundStartL1Block) } // Get the active transcoder pool size when we receive a NewRound event @@ -264,3 +294,7 @@ func (tw *TimeWatcher) handleLog(log types.Log) error { return nil } + +func prevRound(lr *big.Int) *big.Int { + return big.NewInt(0).Sub(lr, big.NewInt(1)) +} diff --git a/eth/watchers/timewatcher_test.go b/eth/watchers/timewatcher_test.go index 73f6123..520b780 100644 --- a/eth/watchers/timewatcher_test.go +++ b/eth/watchers/timewatcher_test.go @@ -2,6 +2,7 @@ package watchers import ( "fmt" + "github.com/livepeer/go-livepeer/eth/contracts" "math/big" "testing" "time" @@ -21,15 +22,20 @@ func TestSetAndGet_LastInitializedRound_LastInitializedBlockHash(t *testing.T) { round := big.NewInt(5) var hash [32]byte copy(hash[:], "hello world") + var preHash [32]byte + copy(preHash[:], "hey hey hey") num := big.NewInt(10) - tw.setLastInitializedRound(round, hash, num) + tw.setLastInitializedRound(round, hash, preHash, num) assert.Equal(tw.lastInitializedRound, round) assert.Equal(tw.lastInitializedL1BlockHash, hash) + assert.Equal(tw.preLastInitializedL1BlockHash, preHash) r := tw.LastInitializedRound() assert.Equal(r, round) h := tw.LastInitializedL1BlockHash() assert.Equal(h, hash) + ph := tw.PreLastInitializedL1BlockHash() + assert.Equal(ph, preHash) assert.Equal(tw.CurrentRoundStartL1Block(), num) } @@ -106,10 +112,14 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { assert.EqualError(err, expErr) lpEth.Errors["GetTranscoderPoolSize"] = nil + preHash := ethcommon.HexToHash("bar") + lpEth.BlockHashForRoundMap = map[int64]common.Hash{0: preHash, 1: hash} tw, err = NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) assert.Nil(err) bh := tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) + pbh := tw.PreLastInitializedL1BlockHash() + assert.Equal(preHash, common.BytesToHash(pbh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(block, tw.LastSeenL1Block()) @@ -120,6 +130,8 @@ func TestTimeWatcher_NewTimeWatcher(t *testing.T) { assert.Nil(err) bh = tw.LastInitializedL1BlockHash() assert.Equal(hash, common.BytesToHash(bh[:])) + pbh = tw.PreLastInitializedL1BlockHash() + assert.Equal(preHash, common.BytesToHash(pbh[:])) assert.Equal(round, tw.LastInitializedRound()) assert.Equal(size, tw.GetTranscoderPoolSize()) assert.Equal(big.NewInt(0), tw.LastSeenL1Block()) @@ -199,7 +211,7 @@ func TestTimeWatcher_WatchAndStop(t *testing.T) { } func TestTimeWatcher_HandleLog(t *testing.T) { - lpEth := ð.StubClient{} + lpEth := ð.StubClient{Round: big.NewInt(2)} watcher := &stubBlockWatcher{} tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) require.Nil(t, err) @@ -212,10 +224,76 @@ func TestTimeWatcher_HandleLog(t *testing.T) { err = tw.handleLog(log) assert.Nil(err) - assert.Nil(tw.LastInitializedRound()) + assert.Equal(big.NewInt(2), tw.LastInitializedRound()) assert.Equal([32]byte{}, tw.LastInitializedL1BlockHash()) } +func TestTimeWatcher_HandleDecodedLog(t *testing.T) { + assert := assert.New(t) + + lpEth := ð.StubClient{ + Round: big.NewInt(2), + BlockHashForRoundMap: map[int64]common.Hash{ + 1: ethcommon.BytesToHash([]byte("one")), + 2: ethcommon.BytesToHash([]byte("two")), + 3: ethcommon.BytesToHash([]byte("three")), + 4: ethcommon.BytesToHash([]byte("four")), + 5: ethcommon.BytesToHash([]byte("five")), + 6: ethcommon.BytesToHash([]byte("six")), + }, + } + watcher := &stubBlockWatcher{} + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, lpEth) + require.Nil(t, err) + + var hash [32]byte + var preHash [32]byte + var nr contracts.RoundsManagerNewRound + + // Test remove log, data refreshed from eth client + log := newStubBaseLog() + log.Removed = true + lpEth.Round = big.NewInt(3) + + err = tw.handleDecodedLog(log, nr) + + assert.Nil(err) + assert.Equal(big.NewInt(3), tw.LastInitializedRound()) + copy(hash[:], lpEth.BlockHashForRoundMap[3].Bytes()) + assert.Equal(hash, tw.LastInitializedL1BlockHash()) + copy(preHash[:], lpEth.BlockHashForRoundMap[2].Bytes()) + assert.Equal(preHash, tw.PreLastInitializedL1BlockHash()) + + // Test add log with subsequent round + log.Removed = false + currentHash := ethcommon.BytesToHash([]byte("last")) + tw.lastInitializedL1BlockHash = currentHash + nr.Round = big.NewInt(4) + nr.BlockHash = lpEth.BlockHashForRoundMap[4] + + err = tw.handleDecodedLog(log, nr) + + assert.Nil(err) + assert.Equal(big.NewInt(4), tw.LastInitializedRound()) + copy(hash[:], lpEth.BlockHashForRoundMap[4].Bytes()) + assert.Equal(hash, tw.LastInitializedL1BlockHash()) + copy(preHash[:], currentHash.Bytes()) + assert.Equal(preHash, tw.PreLastInitializedL1BlockHash()) + + // Test add log with not subsequent round + nr.Round = big.NewInt(6) + nr.BlockHash = lpEth.BlockHashForRoundMap[6] + + err = tw.handleDecodedLog(log, nr) + + assert.Nil(err) + assert.Equal(big.NewInt(6), tw.LastInitializedRound()) + copy(hash[:], lpEth.BlockHashForRoundMap[6].Bytes()) + assert.Equal(hash, tw.LastInitializedL1BlockHash()) + copy(preHash[:], lpEth.BlockHashForRoundMap[5].Bytes()) + assert.Equal(preHash, tw.PreLastInitializedL1BlockHash()) +} + func TestLastSeenBlock(t *testing.T) { assert := assert.New(t) tw := &TimeWatcher{} @@ -232,7 +310,7 @@ func TestHandleBlockNum(t *testing.T) { latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } - tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{Round: big.NewInt(2)}) assert.Nil(err) header := defaultMiniHeader() header.L1BlockNumber = big.NewInt(10) @@ -255,7 +333,7 @@ func TestSubscribeBlocks(t *testing.T) { latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } - tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{Round: big.NewInt(2)}) assert.Nil(err) header := defaultMiniHeader() header.L1BlockNumber = big.NewInt(10) @@ -283,7 +361,7 @@ func TestSubscribeRounds(t *testing.T) { latestHeader: &blockwatch.MiniHeader{L1BlockNumber: big.NewInt(1)}, } - tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{}) + tw, err := NewTimeWatcher(stubRoundsManagerAddr, watcher, ð.StubClient{Round: big.NewInt(2)}) assert.Nil(err) header := defaultMiniHeader() newRoundEvent := newStubNewRoundLog() From a03c6a4ba67c460c01ef41506d1c9bfdd7a99a54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Mon, 11 Jul 2022 12:23:51 +0200 Subject: [PATCH 69/72] eth: Backfill blocks in batches (#2489) Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 159deaa162065f933ac472d11b05510f829d1692 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 40 ++++++++++++++++------------ eth/blockwatch/block_watcher_test.go | 7 ++--- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index c097b58..bc93277 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -85,10 +85,12 @@ func New(config Config) *Watcher { return bs } -// BackfillEventsIfNeeded finds missed events that might have occured while the -// node was offline and sends them to event subscribers. It blocks until -// it is done backfilling or the given context is canceled. -func (w *Watcher) BackfillEventsIfNeeded(ctx context.Context) error { +// BackfillEvents finds missed events and sends them to event subscribers. +// It blocks until it is done backfilling or the given context is canceled. +// Note that the latest block is never backfilled here from logs. It will be polled separately in syncToLatestBlock(). +// The reason for that is that we always need to propagate events from the latest block even if it does not contain +// events which are filtered out during the backfilling process. +func (w *Watcher) BackfillEvents(ctx context.Context) error { events, err := w.getMissedEventsToBackfill(ctx) if err != nil { return err @@ -117,7 +119,7 @@ func (w *Watcher) Watch(ctx context.Context) error { ticker.Stop() return nil case <-ticker.C: - if err := w.syncToLatestBlock(); err != nil { + if err := w.syncToLatestBlock(ctx); err != nil { glog.Errorf("blockwatch.Watcher error encountered - trying again on next polling interval err=%q", err) } } @@ -147,9 +149,14 @@ func (w *Watcher) InspectRetainedBlocks() ([]*MiniHeader, error) { return w.stack.Inspect() } -func (w *Watcher) syncToLatestBlock() error { +func (w *Watcher) syncToLatestBlock(ctx context.Context) error { w.Lock() defer w.Unlock() + + if err := w.BackfillEvents(ctx); err != nil { + return err + } + newestHeader, err := w.client.HeaderByNumber(nil) if err != nil { return err @@ -303,10 +310,11 @@ func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { return header, nil } -// getMissedEventsToBackfill finds missed events that might have occured while the node was -// offline. It does this by comparing the last block stored with the latest block discoverable via RPC. -// If the stored block is older then the latest block, it batch fetches the events for missing blocks, +// getMissedEventsToBackfill finds missed events that might have occurred since the last block polling. +// It does this by comparing the last block stored with the latest block discoverable via RPC. +// If the stored block is older than the latest block, it batch-fetches the events for missing blocks, // re-sets the stored blocks and returns the block events found. +// Note that the latest block is never backfilled, and will be polled separately during in syncToLatestBlock(). func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, error) { events := []*Event{} @@ -325,7 +333,9 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro if err != nil { return events, err } - latestBlockNum := int(latestBlock.Number.Int64()) + + // Latest block will be polled separately in syncToLatestBlock(), so it's not backfilled. + preLatestBlockNum := int(latestBlock.Number.Int64()) - 1 if latestRetainedBlock != nil { latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) @@ -335,14 +345,11 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro return events, nil } - if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed <= 0 { + if blocksElapsed = preLatestBlockNum - startBlockNum; blocksElapsed <= 0 { return events, nil } - glog.Infof("Backfilling block events (this can take a while)...\n") - glog.Infof("Start block: %v End block: %v Blocks elapsed: %v\n", startBlockNum, startBlockNum+blocksElapsed, blocksElapsed) - - logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, latestBlockNum) + logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, preLatestBlockNum) if furthestBlockProcessed > latestRetainedBlockNum { // If we have processed blocks further then the latestRetainedBlock in the DB, we // want to remove all blocks from the DB and insert the furthestBlockProcessed @@ -396,7 +403,6 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro BlockHeader: blockHeader, }) } - glog.Info("Done backfilling block events") return events, nil } return events, nil @@ -558,7 +564,7 @@ func (w *Watcher) getSubBlockRanges(from, to, rangeSize int) []*blockRange { const infuraTooManyResultsErrMsg = "query returned more than 10000 results" func (w *Watcher) filterLogsRecursively(from, to int, allLogs []types.Log) ([]types.Log, error) { - glog.Infof("fetching block logs from=%v to=%v", from, to) + glog.V(6).Infof("Polling blocks from=%v to=%v", from, to) numBlocks := to - from topics := [][]common.Hash{} if len(w.topics) > 0 { diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index f94f68b..47ac137 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -189,12 +189,13 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { store := &stubMiniHeaderStore{} // Add block number 5 as the last block seen by BlockWatcher - lastBlockSeen := &MiniHeader{ + preLastBlockSeen := &MiniHeader{ Number: big.NewInt(5), Hash: common.HexToHash("0x293b9ea024055a3e9eddbf9b9383dc7731744111894af6aa038594dc1b61f87f"), Parent: common.HexToHash("0x26b13ac89500f7fcdd141b7d1b30f3a82178431eca325d1cf10998f9d68ff5ba"), } - err = store.InsertMiniHeader(lastBlockSeen) + err = store.InsertMiniHeader(preLastBlockSeen) + require.NoError(t, err) config.Store = store @@ -211,7 +212,7 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { headers, err := store.FindAllMiniHeadersSortedByNumber() require.NoError(t, err) require.Len(t, headers, 1) - assert.Equal(t, big.NewInt(30), headers[0].Number) + assert.Equal(t, big.NewInt(29), headers[0].Number) } func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { From 907247e74206aa2c709b819c358bb7add90edf0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Leszko?= Date: Thu, 23 Mar 2023 07:35:18 +0100 Subject: [PATCH 70/72] Reduce number of ETH RPC calls during block polling (#2775) Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 8c7ec0155c5ba040f7066ab0a30045c37f15be2a Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 181 +++++---------------------- eth/blockwatch/block_watcher_test.go | 17 ++- 2 files changed, 41 insertions(+), 157 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index bc93277..3f49ab2 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -20,7 +20,7 @@ import ( // a hard limit of 10,000 logs returned by a single `eth_getLogs` query by Infura's Ethereum nodes so // we need to try and stay below it. Parity, Geth and Alchemy all have much higher limits (if any) on // the number of logs returned so Infura is by far the limiting factor. -var maxBlocksInGetLogsQuery = 60 +var maxBlocksInGetLogsQuery = 1000 // EventType describes the types of events emitted by blockwatch.Watcher. A block can be discovered // and added to our representation of the chain. During a block re-org, a block previously stored @@ -90,8 +90,8 @@ func New(config Config) *Watcher { // Note that the latest block is never backfilled here from logs. It will be polled separately in syncToLatestBlock(). // The reason for that is that we always need to propagate events from the latest block even if it does not contain // events which are filtered out during the backfilling process. -func (w *Watcher) BackfillEvents(ctx context.Context) error { - events, err := w.getMissedEventsToBackfill(ctx) +func (w *Watcher) BackfillEvents(ctx context.Context, chainHead *MiniHeader) error { + var events, err = w.getMissedEventsToBackfill(ctx, chainHead) if err != nil { return err } @@ -153,146 +153,12 @@ func (w *Watcher) syncToLatestBlock(ctx context.Context) error { w.Lock() defer w.Unlock() - if err := w.BackfillEvents(ctx); err != nil { - return err - } - newestHeader, err := w.client.HeaderByNumber(nil) if err != nil { return err } - lastSeenHeader, err := w.stack.Peek() - if err != nil { - return err - } - - if lastSeenHeader == nil { - return w.pollNextBlock() - } - - for i := lastSeenHeader.Number; i.Cmp(newestHeader.Number) < 0; i = i.Add(i, big.NewInt(1)) { - if err := w.pollNextBlock(); err != nil { - return err - } - } - return nil -} - -// pollNextBlock polls for the next block header to be added to the block stack. -// If there are no blocks on the stack, it fetches the first block at the specified -// `startBlockDepth` supplied at instantiation. -func (w *Watcher) pollNextBlock() error { - var nextBlockNumber *big.Int - latestHeader, err := w.stack.Peek() - if err != nil { - return err - } - if latestHeader == nil { - if w.startBlockDepth == rpc.LatestBlockNumber { - nextBlockNumber = nil // Fetch latest block - } else { - nextBlockNumber = big.NewInt(int64(w.startBlockDepth)) - } - } else { - nextBlockNumber = big.NewInt(0).Add(latestHeader.Number, big.NewInt(1)) - } - nextHeader, err := w.client.HeaderByNumber(nextBlockNumber) - if err != nil { - if err == ethereum.NotFound { - return nil // Noop and wait next polling interval - } - return err - } - - events := []*Event{} - events, err = w.buildCanonicalChain(nextHeader, events) - // Even if an error occurred, we still want to emit the events gathered since we might have - // popped blocks off the Stack and they won't be re-added - if len(events) != 0 { - w.blockFeed.Send(w.enrichWithL1(events)) - } - if err != nil { - return err - } - return nil -} - -func (w *Watcher) buildCanonicalChain(nextHeader *MiniHeader, events []*Event) ([]*Event, error) { - latestHeader, err := w.stack.Peek() - if err != nil { - return nil, err - } - // Is the stack empty or is it the next block? - if latestHeader == nil || nextHeader.Parent == latestHeader.Hash { - nextHeader, err := w.addLogs(nextHeader) - if err != nil { - // Due to block re-orgs & Ethereum node services load-balancing requests across multiple nodes - // a block header might be returned, but when fetching it's logs, an "unknown block" error is - // returned. This is expected to happen sometimes, and we simply return the events gathered so - // far and pick back up where we left off on the next polling interval. - if isUnknownBlockErr(err) { - glog.V(5).Infof("missing logs for blockNumber=%v - fetching on next polling interval", nextHeader.Number) - return events, nil - } - return events, err - } - err = w.stack.Push(nextHeader) - if err != nil { - return events, err - } - events = append(events, &Event{ - Type: Added, - BlockHeader: nextHeader, - }) - return events, nil - } - - // Pop latestHeader from the stack. We already have a reference to it. - if _, err := w.stack.Pop(); err != nil { - return events, err - } - events = append(events, &Event{ - Type: Removed, - BlockHeader: latestHeader, - }) - - nextParentHeader, err := w.client.HeaderByHash(nextHeader.Parent) - if err != nil { - if err == ethereum.NotFound { - glog.V(5).Infof("block header not found blockHash=%v - fetching on next polling interval", nextHeader.Parent.Hex()) - // Noop and wait next polling interval. We remove the popped blocks - // and refetch them on the next polling interval. - return events, nil - } - return events, err - } - events, err = w.buildCanonicalChain(nextParentHeader, events) - if err != nil { - return events, err - } - nextHeader, err = w.addLogs(nextHeader) - if err != nil { - // Due to block re-orgs & Ethereum node services load-balancing requests across multiple nodes - // a block header might be returned, but when fetching it's logs, an "unknown block" error is - // returned. This is expected to happen sometimes, and we simply return the events gathered so - // far and pick back up where we left off on the next polling interval. - if isUnknownBlockErr(err) { - glog.V(5).Infof("missing logs for blockNumber=%v - fetching on next polling interval", nextHeader.Number) - return events, nil - } - return events, err - } - err = w.stack.Push(nextHeader) - if err != nil { - return events, err - } - events = append(events, &Event{ - Type: Added, - BlockHeader: nextHeader, - }) - - return events, nil + return w.BackfillEvents(ctx, newestHeader) } func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { @@ -315,7 +181,7 @@ func (w *Watcher) addLogs(header *MiniHeader) (*MiniHeader, error) { // If the stored block is older than the latest block, it batch-fetches the events for missing blocks, // re-sets the stored blocks and returns the block events found. // Note that the latest block is never backfilled, and will be polled separately during in syncToLatestBlock(). -func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, error) { +func (w *Watcher) getMissedEventsToBackfill(ctx context.Context, chainHead *MiniHeader) ([]*Event, error) { events := []*Event{} var ( @@ -329,27 +195,38 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro return events, err } - latestBlock, err := w.client.HeaderByNumber(nil) - if err != nil { - return events, err + if chainHead == nil { + var err error + chainHead, err = w.client.HeaderByNumber(nil) + if err != nil { + return events, err + } } - // Latest block will be polled separately in syncToLatestBlock(), so it's not backfilled. - preLatestBlockNum := int(latestBlock.Number.Int64()) - 1 + latestBlockNum := int(chainHead.Number.Int64()) if latestRetainedBlock != nil { latestRetainedBlockNum = int(latestRetainedBlock.Number.Int64()) // Events for latestRetainedBlock already processed, start at latestRetainedBlock + 1 startBlockNum = latestRetainedBlockNum + 1 } else { + err = w.stack.Push(chainHead) + if err != nil { + return events, err + } + + events = append(events, &Event{ + Type: Added, + BlockHeader: chainHead, + }) return events, nil } - if blocksElapsed = preLatestBlockNum - startBlockNum; blocksElapsed <= 0 { + if blocksElapsed = latestBlockNum - startBlockNum; blocksElapsed <= 0 { return events, nil } - logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, preLatestBlockNum) + logs, furthestBlockProcessed := w.getLogsInBlockRange(ctx, startBlockNum, latestBlockNum) if furthestBlockProcessed > latestRetainedBlockNum { // If we have processed blocks further then the latestRetainedBlock in the DB, we // want to remove all blocks from the DB and insert the furthestBlockProcessed @@ -365,7 +242,12 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro } } // Add furthest block processed into the DB - latestHeader, err := w.client.HeaderByNumber(big.NewInt(int64(furthestBlockProcessed))) + var latestHeader *MiniHeader + if chainHead.Number.Int64() == int64(furthestBlockProcessed) { + latestHeader = chainHead + } else { + latestHeader, err = w.client.HeaderByNumber(big.NewInt(int64(furthestBlockProcessed))) + } if err != nil { return events, err } @@ -374,6 +256,11 @@ func (w *Watcher) getMissedEventsToBackfill(ctx context.Context) ([]*Event, erro return events, err } + events = append(events, &Event{ + Type: Added, + BlockHeader: latestHeader, + }) + // If no logs found, noop if len(logs) == 0 { return events, nil diff --git a/eth/blockwatch/block_watcher_test.go b/eth/blockwatch/block_watcher_test.go index 47ac137..12c41d0 100644 --- a/eth/blockwatch/block_watcher_test.go +++ b/eth/blockwatch/block_watcher_test.go @@ -41,9 +41,6 @@ func TestWatcher(t *testing.T) { for i := 0; i < fakeClient.NumberOfTimesteps(); i++ { scenarioLabel := fakeClient.GetScenarioLabel() - err := watcher.pollNextBlock() - require.NoError(t, err) - retainedBlocks, err := watcher.InspectRetainedBlocks() require.NoError(t, err) expectedRetainedBlocks := fakeClient.ExpectedRetainedBlocks() @@ -204,15 +201,15 @@ func TestGetMissedEventsToBackfillSomeMissed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - events, err := watcher.getMissedEventsToBackfill(ctx) + events, err := watcher.getMissedEventsToBackfill(ctx, nil) require.NoError(t, err) - assert.Len(t, events, 1) + assert.Len(t, events, 2) // Check that block 30 is now in the DB, and block 5 was removed. headers, err := store.FindAllMiniHeadersSortedByNumber() require.NoError(t, err) require.Len(t, headers, 1) - assert.Equal(t, big.NewInt(29), headers[0].Number) + assert.Equal(t, big.NewInt(30), headers[0].Number) } func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { @@ -236,7 +233,7 @@ func TestGetMissedEventsToBackfillNoneMissed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - events, err := watcher.getMissedEventsToBackfill(ctx) + events, err := watcher.getMissedEventsToBackfill(ctx, nil) require.NoError(t, err) assert.Len(t, events, 0) @@ -262,13 +259,13 @@ func TestGetMissedEventsToBackfill_NOOP(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - events, err := watcher.getMissedEventsToBackfill(ctx) + events, err := watcher.getMissedEventsToBackfill(ctx, nil) require.NoError(t, err) - assert.Len(t, events, 0) + assert.Len(t, events, 1) headers, err := store.FindAllMiniHeadersSortedByNumber() require.NoError(t, err) - require.Len(t, headers, 0) + require.Len(t, headers, 1) } var logStub = types.Log{ From 5d60aa83ff762acb471cfb614433bdfdc79cae22 Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Tue, 6 Jun 2023 13:42:28 -0300 Subject: [PATCH 71/72] go.mod: Bump go and livepeer-data version (#2804) * go.mod: Update livepeer-data lib * server: Fix MetadataQueue type Type changed on livepeer-data version * go.mod: Bump go to 1.20 The upgrade indirectly added a module that requires 1.19 * changelog: Change changelog * Go 1.20 -> 1.20.4 * Get rid of go-version-file * Bump lint version * Set go version in another place * go fmt * Add glog goroutine to ignored leak ones * Change order of go mod command * Bump linter version --------- Co-authored-by: Thom Shutt Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 1b9da7f25fa8c6261e8c465ea7e5c75166d220b6 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/watchers/timewatcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/watchers/timewatcher.go b/eth/watchers/timewatcher.go index 4fbcd24..45acfe2 100644 --- a/eth/watchers/timewatcher.go +++ b/eth/watchers/timewatcher.go @@ -22,8 +22,8 @@ import ( // TimeWatcher allows for subscriptions to certain data feeds using a caller provided sink channel // consumers of the TimeWatcher can subscribe to following data feeds: -// * Last Initialized Round Number -// * Last Seen Block Number +// - Last Initialized Round Number +// - Last Seen Block Number type TimeWatcher struct { // state mu sync.RWMutex From cd1df2c8d03ec053fdc04803b1275d73ddd40253 Mon Sep 17 00:00:00 2001 From: studystill <137779852+studystill@users.noreply.github.com> Date: Tue, 22 Jul 2025 15:39:36 +0800 Subject: [PATCH 72/72] refactor: use the built-in max/min to simplify the code (#3577) Signed-off-by: studystill Source-Repository: https://github.com/livepeer/go-livepeer Original-Commit: 3a69e439b60acbcfb8edfda15896fb1c25039958 Source-Scope: eth/watchers/roundswatcher.go; eth/watchers/roundswatcher_test.go; eth/watchers/timewatcher.go; eth/watchers/timewatcher_test.go; eth/blockwatch/block_watcher.go; eth/blockwatch/block_watcher_test.go --- eth/blockwatch/block_watcher.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/eth/blockwatch/block_watcher.go b/eth/blockwatch/block_watcher.go index 3f49ab2..32a9772 100644 --- a/eth/blockwatch/block_watcher.go +++ b/eth/blockwatch/block_watcher.go @@ -433,10 +433,7 @@ func (w *Watcher) getSubBlockRanges(from, to, rangeSize int) []*blockRange { for i := 0; i < numChunks; i = i + 1 { fromIndex := i * rangeSize - toIndex := fromIndex + rangeSize - if toIndex > len(blocks) { - toIndex = len(blocks) - } + toIndex := min(fromIndex+rangeSize, len(blocks)) bs := blocks[fromIndex:toIndex] blockRange := &blockRange{ FromBlock: bs[0],