I was trying to count all my STEEM/SBD balances for my tax report as I found out that the STEEM account history api is not reliable:
#!/usr/bin/python
from beem import Steem
from beem.account import Account
from beem.amount import Amount
from beem.nodelist import NodeList
if __name__ == "__main__":
nodelist = NodeList()
nodelist.update_nodes()
stm = Steem(node=nodelist.get_steem_nodes())
print(stm)
account_name = "holger80"
account = Account(account_name, steem_instance=stm)
ops_dict = {}
for ops in account.history():
ops_dict[ops["index"]] = ops
STEEM = 0
SBD = 0
index = 0
for _id in sorted(list(ops_dict.keys())):
ops = ops_dict[_id]
if ops["type"] == "fill_convert_request":
amount_out = Amount(ops["amount_out"], blockchain_instance=stm)
amount_in = Amount(ops["amount_in"], blockchain_instance=stm)
STEEM += float(amount_out)
SBD -= float(amount_in)
elif ops["type"] == "fill_transfer_from_savings":
amount = Amount(ops["amount"], steem_instance=stm)
if ops["to"] == account_name:
if amount.symbol == "STEEM":
STEEM += float(amount)
else:
SBD += float(amount)
elif ops["type"] == "transfer_to_savings":
amount = Amount(ops["amount"], steem_instance=stm)
if amount.symbol == "STEEM":
STEEM -= float(amount)
else:
SBD -= float(amount)
elif ops["type"] == "transfer_to_vesting":
amount = Amount(ops["amount"], steem_instance=stm)
if ops["from"] == account_name and ops["to"] == account_name:
STEEM -= float(amount)
elif ops["from"] == account_name:
STEEM -= float(amount)
index += 1
elif ops["type"] == "fill_vesting_withdraw":
amount = Amount(ops["deposited"], steem_instance=stm)
if ops["to_account"] == account_name:
STEEM += float(amount)
elif ops["type"] == "proposal_pay":
amount = Amount(ops["payment"], steem_instance=stm)
SBD += float(amount)
index += 1
elif ops["type"] == "create_proposal":
SBD -= 10
index += 1
elif ops["type"] == "transfer":
amount = Amount(ops["amount"], steem_instance=stm)
if ops["from"] == account_name and ops["to"] == account_name:
continue
if ops["to"] == account_name:
if amount.symbol == "STEEM":
STEEM += float(amount)
else:
SBD += float(amount)
else:
if amount.symbol == "STEEM":
STEEM -= float(amount)
else:
SBD -= float(amount)
index += 1
elif ops["type"] == "account_create_with_delegation":
if ops["new_account_name"] == account_name:
continue
fee = Amount(ops["fee"], blockchain_instance=stm)
STEEM -= float(fee)
elif ops["type"] == "account_create":
if ops["new_account_name"] == account_name:
continue
fee = Amount(ops["fee"], blockchain_instance=stm)
STEEM -= float(fee)
elif ops["type"] == "claim_reward_balance":
reward_steem = Amount(ops["reward_steem"], steem_instance=stm)
reward_vests = Amount(ops["reward_vests"], steem_instance=stm)
reward_sbd = Amount(ops["reward_sbd"], steem_instance=stm)
if float(reward_steem) > 0:
STEEM += float(reward_steem)
if float(reward_sbd) > 0:
SBD += float(reward_sbd)
index += 1
elif ops["type"] == "fill_order":
open_pays = Amount(ops["open_pays"], steem_instance=stm)
current_pays = Amount(ops["current_pays"], steem_instance=stm)
open_owner = ops["open_owner"]
current_owner = ops["current_owner"]
if current_owner == account_name:
if open_pays.symbol == "STEEM":
STEEM += float(open_pays)
SBD -= float(current_pays)
else:
SBD += float(open_pays)
STEEM -= float(current_pays)
else:
if current_pays.symbol == "STEEM":
STEEM += float(current_pays)
SBD -= float(open_pays)
else:
SBD += float(current_pays)
STEEM -= float(open_pays)
index += 1
print("%.3f STEEM, %.3f SBD" % (STEEM, SBD))
returns
6115.776 STEEM, -3.775 SBD
which is completely wrong. First it cannot be negative and second steemd.com shows
After investigating, I found out that some account history elements are shown twice.
1. check: Works beem as it should?
E.g. beem returns:
print(ops_dict[150050])
print(ops_dict[150053])
{'current_owner': 'holger80', 'current_orderid': 1584865440, 'current_pays': {'amount': '3775', 'precision': 3, 'nai': '@@000000013'}, 'open_owner': 'cst90', 'open_orderid': 1584865425, 'open_pays': {'amount': '20000', 'precision': 3, 'nai': '@@000000021'}, 'trx_id': '0a3200e502bd6b1c06b81c2a2337e0436fc13704', 'block': 41868606, 'trx_in_block': 24, 'op_in_trx': 0, 'virtual_op': 1, 'timestamp': '2020-03-22T08:24:00', 'account': 'holger80', 'type': 'fill_order', '_id': '60ccd83432667fccb51946202811a6d65f68a6ea', 'index': 150050}
{'current_owner': 'holger80', 'current_orderid': 1584865440, 'current_pays': {'amount': '3775', 'precision': 3, 'nai': '@@000000013'}, 'open_owner': 'cst90', 'open_orderid': 1584865425, 'open_pays': {'amount': '20000', 'precision': 3, 'nai': '@@000000021'}, 'trx_id': '0a3200e502bd6b1c06b81c2a2337e0436fc13704', 'block': 41868606, 'trx_in_block': 24, 'op_in_trx': 0, 'virtual_op': 1, 'timestamp': '2020-03-22T08:24:00', 'account': 'holger80', 'type': 'fill_order', '_id': '60ccd83432667fccb51946202811a6d65f68a6ea', 'index': 150053}
Which is also shown on steemd.com:
The error is not caused by beem.
2. check: What is stored in the trx id?
https://steemd.com/tx/0a3200e502bd6b1c06b81c2a2337e0436fc13704 retuns only one operation:
3. check: What shows the block?
https://steemd.com/b/41868606#0a3200e502bd6b1c06b81c2a2337e0436fc13704
shows that there is only one operation:
How to fix?
I'm using the internal _id parameter from beem for this. For every operation, a unique identifier is calculated. Whenever when two operations are identically, their _id values are identically. I use this _id to find all duplicate entries and count then all operations with the same transaction id in the block. When there are more entries than counted operations, I mark the surplus entries as duplicate.
#!/usr/bin/python
from beem import Steem
from beem.block import Block
from beem.account import Account
from beem.amount import Amount
from beem.nodelist import NodeList
if __name__ == "__main__":
nodelist = NodeList()
nodelist.update_nodes()
stm = Steem(node=nodelist.get_steem_nodes())
print(stm)
account_name = "holger80"
account = Account(account_name, steem_instance=stm)
ops_dict = {}
_ids = {}
for ops in account.history():
ops_dict[ops["index"]] = ops
if ops["_id"] in _ids:
_ids[ops["_id"]] += 1
else:
_ids[ops["_id"]] = 1
duplicate_indices = []
_id_list = []
for _id in sorted(list(ops_dict.keys())):
ops = ops_dict[_id]
if _ids[ops["_id"]] == 1:
continue
if ops["_id"] not in _id_list:
_id_list.append(ops["_id"])
else:
trx_id = ops["trx_id"]
if trx_id == "0000000000000000000000000000000000000000":
duplicate_indices.append(ops["index"])
else:
block = Block(ops["block"], blockchain_instance=stm)
count_ops = 0
for t in block.transactions:
if t["transaction_id"] != trx_id:
continue
for o in t["operations"]:
count_ops += 1
if count_ops < _ids[ops["_id"]]:
duplicate_indices.append(ops["index"])
STEEM = 0
SBD = 0
index = 0
print("duplicate indices %d" % len(duplicate_indices))
for _id in sorted(list(ops_dict.keys())):
ops = ops_dict[_id]
if _id in duplicate_indices:
continue
if ops["type"] == "fill_convert_request":
amount_out = Amount(ops["amount_out"], blockchain_instance=stm)
amount_in = Amount(ops["amount_in"], blockchain_instance=stm)
STEEM += float(amount_out)
SBD -= float(amount_in)
index += 1
elif ops["type"] == "fill_transfer_from_savings":
amount = Amount(ops["amount"], steem_instance=stm)
if ops["to"] == account_name:
if amount.symbol == "STEEM":
STEEM += float(amount)
else:
SBD += float(amount)
index += 1
elif ops["type"] == "transfer_to_savings":
amount = Amount(ops["amount"], steem_instance=stm)
if amount.symbol == "STEEM":
STEEM -= float(amount)
else:
SBD -= float(amount)
index += 1
elif ops["type"] == "transfer_to_vesting":
amount = Amount(ops["amount"], steem_instance=stm)
if ops["from"] == account_name and ops["to"] == account_name:
STEEM -= float(amount)
elif ops["from"] == account_name:
STEEM -= float(amount)
index += 1
elif ops["type"] == "fill_vesting_withdraw":
amount = Amount(ops["deposited"], steem_instance=stm)
if ops["to_account"] == account_name:
STEEM += float(amount)
index += 1
elif ops["type"] == "proposal_pay":
amount = Amount(ops["payment"], steem_instance=stm)
SBD += float(amount)
index += 1
elif ops["type"] == "create_proposal":
SBD -= 10
index += 1
elif ops["type"] == "transfer":
amount = Amount(ops["amount"], steem_instance=stm)
if ops["from"] == account_name and ops["to"] == account_name:
continue
if ops["to"] == account_name:
if amount.symbol == "STEEM":
STEEM += float(amount)
else:
SBD += float(amount)
else:
if amount.symbol == "STEEM":
STEEM -= float(amount)
else:
SBD -= float(amount)
index += 1
elif ops["type"] == "account_create_with_delegation":
if ops["new_account_name"] == account_name:
continue
fee = Amount(ops["fee"], blockchain_instance=stm)
STEEM -= float(fee)
index += 1
elif ops["type"] == "account_create":
if ops["new_account_name"] == account_name:
continue
fee = Amount(ops["fee"], blockchain_instance=stm)
STEEM -= float(fee)
index += 1
elif ops["type"] == "claim_reward_balance":
reward_steem = Amount(ops["reward_steem"], steem_instance=stm)
reward_vests = Amount(ops["reward_vests"], steem_instance=stm)
reward_sbd = Amount(ops["reward_sbd"], steem_instance=stm)
if float(reward_steem) > 0:
STEEM += float(reward_steem)
if float(reward_sbd) > 0:
SBD += float(reward_sbd)
index += 1
elif ops["type"] == "fill_order":
open_pays = Amount(ops["open_pays"], steem_instance=stm)
current_pays = Amount(ops["current_pays"], steem_instance=stm)
open_owner = ops["open_owner"]
current_owner = ops["current_owner"]
if current_owner == account_name:
if open_pays.symbol == "STEEM":
STEEM += float(open_pays)
SBD -= float(current_pays)
else:
SBD += float(open_pays)
STEEM -= float(current_pays)
else:
if current_pays.symbol == "STEEM":
STEEM += float(current_pays)
SBD -= float(open_pays)
else:
SBD += float(current_pays)
STEEM -= float(open_pays)
index += 1
print("%d entries" % index)
print("%.3f STEEM, %.3f SBD" % (STEEM, SBD))
This time, the script returns:
duplicate indices 26
0.001 STEEM, 0.000 SBD
which is now correct. After removing 26 duplicate entries, my balance is correct.
Is there also a problem with duplicate entries on Hive?
#!/usr/bin/python
from beem import Hive
from beem.account import Account
from beem.amount import Amount
from beem.block import Block
from beem.nodelist import NodeList
if __name__ == "__main__":
nodelist = NodeList()
nodelist.update_nodes()
stm = Hive(node=nodelist.get_hive_nodes())
print(stm)
account_name = "holger80"
account = Account(account_name, blockchain_instance=stm)
ops_dict = {}
_ids = {}
for ops in account.history():
ops_dict[ops["index"]] = ops
if ops["_id"] in _ids:
_ids[ops["_id"]] += 1
else:
_ids[ops["_id"]] = 1
duplicate_indices = []
_id_list = []
for _id in sorted(list(ops_dict.keys())):
ops = ops_dict[_id]
if _ids[ops["_id"]] == 1:
continue
if ops["_id"] not in _id_list:
_id_list.append(ops["_id"])
else:
trx_id = ops["trx_id"]
if trx_id == "0000000000000000000000000000000000000000":
duplicate_indices.append(ops["index"])
else:
block = Block(ops["block"], blockchain_instance=stm)
count_ops = 0
for t in block.transactions:
if t["transaction_id"] != trx_id:
continue
for o in t["operations"]:
count_ops += 1
if count_ops < _ids[ops["_id"]]:
duplicate_indices.append(ops["index"])
type_count = {}
for _id in sorted(list(ops_dict.keys())):
ops = ops_dict[_id]
if ops["type"] in type_count:
type_count[ops["type"]] += 1
else:
type_count[ops["type"]] = 1
HIVE = 0
HBD = 0
index = 0
print("duplicate indices %d" % len(duplicate_indices))
for _id in sorted(list(ops_dict.keys())):
ops = ops_dict[_id]
if _id in duplicate_indices:
continue
if ops["type"] == "fill_convert_request":
amount_out = Amount(ops["amount_out"], blockchain_instance=stm)
amount_in = Amount(ops["amount_in"], blockchain_instance=stm)
HIVE += float(amount_out)
HBD -= float(amount_in)
index += 1
elif ops["type"] == "fill_transfer_from_savings":
amount = Amount(ops["amount"], blockchain_instance=stm)
if ops["to"] == account_name:
if amount.symbol == "HIVE":
HIVE += float(amount)
else:
HBD += float(amount)
index += 1
elif ops["type"] == "transfer_to_savings":
amount = Amount(ops["amount"], blockchain_instance=stm)
if amount.symbol == "HIVE":
HIVE -= float(amount)
else:
HBD -= float(amount)
index += 1
elif ops["type"] == "transfer_to_vesting":
amount = Amount(ops["amount"], blockchain_instance=stm)
if ops["from"] == account_name and ops["to"] == account_name:
HIVE -= float(amount)
elif ops["from"] == account_name:
HIVE -= float(amount)
index += 1
elif ops["type"] == "fill_vesting_withdraw":
amount = Amount(ops["deposited"], blockchain_instance=stm)
if ops["to_account"] == account_name:
HIVE += float(amount)
index += 1
elif ops["type"] == "proposal_pay":
amount = Amount(ops["payment"], blockchain_instance=stm)
HBD += float(amount)
index += 1
elif ops["type"] == "create_proposal":
HBD -= 10
index += 1
elif ops["type"] == "transfer":
amount = Amount(ops["amount"], blockchain_instance=stm)
if ops["from"] == account_name and ops["to"] == account_name:
continue
if ops["to"] == account_name:
if amount.symbol == "HIVE":
HIVE += float(amount)
else:
HBD += float(amount)
else:
if amount.symbol == "HIVE":
HIVE -= float(amount)
else:
HBD -= float(amount)
index += 1
elif ops["type"] == "account_create_with_delegation":
if ops["new_account_name"] == account_name:
continue
fee = Amount(ops["fee"], blockchain_instance=stm)
HIVE -= float(fee)
index += 1
elif ops["type"] == "account_create":
if ops["new_account_name"] == account_name:
continue
fee = Amount(ops["fee"], blockchain_instance=stm)
HIVE -= float(fee)
index += 1
elif ops["type"] == "claim_reward_balance":
reward_steem = Amount(ops["reward_steem"], blockchain_instance=stm)
reward_vests = Amount(ops["reward_vests"], blockchain_instance=stm)
reward_sbd = Amount(ops["reward_sbd"], blockchain_instance=stm)
if float(reward_steem) > 0:
HIVE += float(reward_steem)
if float(reward_sbd) > 0:
HBD += float(reward_sbd)
index += 1
elif ops["type"] == "fill_order":
open_pays = Amount(ops["open_pays"], blockchain_instance=stm)
current_pays = Amount(ops["current_pays"], blockchain_instance=stm)
open_owner = ops["open_owner"]
current_owner = ops["current_owner"]
if current_owner == account_name:
if open_pays.symbol == "HIVE":
HIVE += float(open_pays)
HBD -= float(current_pays)
else:
HBD += float(open_pays)
HIVE -= float(current_pays)
else:
if current_pays.symbol == "HIVE":
HIVE += float(current_pays)
HBD -= float(open_pays)
else:
HBD += float(current_pays)
HIVE -= float(open_pays)
index += 1
print("%d entries" % index)
print("%.3f HIVE, %.3f HBD" % (HIVE, HBD))
returns
duplicate indices 0
6279 entries
65.316 HIVE, 0.063 HBD
which is correct according to hiveblocks:
Balance is correct and no duplicate entries were found!
Entries that look like duplicate account history entries
First, I thought there is the same problem on Hive as I was tricked by the following two entries:
print(ops_dict[175655])
print(ops_dict[175656])
{'from': 'holger80', 'to': 'beembot', 'amount': {'amount': '1', 'precision': 3, 'nai': '@@000000021'}, 'memo': 'test', 'trx_id': '9b0c144b02d046a18f23639769a8c013bb3fb10a', 'block': 44657346, 'trx_in_block': 9, 'op_in_trx': 0, 'virtual_op': 0, 'timestamp': '2020-06-27T11:50:30', 'account': 'holger80', 'type': 'transfer', '_id': '09fb4d1710c4091f346bcb0fa4e11bfa4128f59c', 'index': 175655}
{'from': 'holger80', 'to': 'beembot', 'amount': {'amount': '1', 'precision': 3, 'nai': '@@000000021'}, 'memo': 'test', 'trx_id': '9b0c144b02d046a18f23639769a8c013bb3fb10a', 'block': 44657346, 'trx_in_block': 9, 'op_in_trx': 0, 'virtual_op': 0, 'timestamp': '2020-06-27T11:50:30', 'account': 'holger80', 'type': 'transfer', '_id': '09fb4d1710c4091f346bcb0fa4e11bfa4128f59c', 'index': 175656}
which are two tranfers in one transaction:
and not a duplicate entry.
print(ops_dict[134132])
print(ops_dict[134133])
returns two times the same custom json:
{'required_auths': [], 'required_posting_auths': ['holger80'], 'id': 'sm_submit_team', 'json': '{"trx_id":"20b319ed22c3fd0986b3c68d4743f86548db631c","team_hash":"66b00a954a974829cc4965255fbdefb6"}', 'trx_id': 'ca950109641db603ee8ebf6f0a4345180643888b', 'block': 36147842, 'trx_in_block': 23, 'op_in_trx': 0, 'virtual_op': 0, 'timestamp': '2019-09-05T05:41:39', 'account': 'holger80', 'type': 'custom_json', '_id': 'e56596adbea0dd4abe434270ba6f6633e30796d9', 'index': 134132}
{'required_auths': [], 'required_posting_auths': ['holger80'], 'id': 'sm_submit_team', 'json': '{"trx_id":"20b319ed22c3fd0986b3c68d4743f86548db631c","team_hash":"66b00a954a974829cc4965255fbdefb6"}', 'trx_id': 'ca950109641db603ee8ebf6f0a4345180643888b', 'block': 36147842, 'trx_in_block': 23, 'op_in_trx': 0, 'virtual_op': 0, 'timestamp': '2019-09-05T05:41:39', 'account': 'holger80', 'type': 'custom_json', '_id': 'e56596adbea0dd4abe434270ba6f6633e30796d9', 'index': 134133}
As there are really two identically operations
(hiveblocks), this is also correct.
Conclusion
There is currently a problem on Steem that the account history API returns sometimes duplicate entries. It is possible to identify them and to remove them.
The account history API on Hive is working correctly.
You can use my scripts to check if your balance can be correctly retraced by the account history api. I will use the scripts and develop them further, so that important account operations can be stored into an excel file.
As the Hive source code is still quite similar to the Steem source code, this may also happen on Hive. In order to prevent this, I created an issue: https://gitlab.syncad.com/hive/hive/-/issues/62
so that that this topic can be explored further.
If you like what I do, consider casting a vote for me as witness on Hivesigner or on PeakD